I'm making a script that needs to move all files from a third folder to the main one and in that list there are also those that are already in the right folder and I need to ignore them.
LIKE THIS: my_bucket/files/id/file.txt TO my_bucket/files/file.txt
my code is like this and it is only listing the objects, it is not moving
I try this code:
defmodule Script.Elixir do
def list_objects_in_bucket do
"my_bucket"
|> ExAws.S3.list_objects(max_keys: 1000)
|> ExAws.request()
|> extract_only_route()
end
def copy_object(origin_path) do
ExAws.S3.put_object_copy(
"my_bucket/",
"/my_folder_destiny/",
"my_bucket",
origin_path
)
|> ExAws.request()
end
defp extract_only_route({:ok, %{body: %{contents: contents}}}) do
Enum.map(contents, fn %{key: route} -> route end)
end
end
Once you have a list of files with list_objects_in_bucket/0, and a tested copy_object/2 function, accepting arguments from and to, you might do somewhat like
#bucket "my_bucket/"
#top "files/"
#nested "id/"
# https://hexdocs.pm/ex_aws_s3/ExAws.S3.html#put_object_copy/5
copy_object = fn from, to ->
#bucket
|> ExAws.S3.put_object_copy(to, #bucket, from)
|> ExAws.request()
end
for #bucket <> #top <> #nested <> name <- list_objects_in_bucket() do
copy_object.(#top <> #nested <> name, #top <> name)
end
Related
Hope everybody is having a beautiful 2019 even though we're just a day in.
I am currently working on a small Phoenix app where I'm manipulating PDF files (in the context of this question I'm splitting them) and then uploading them to S3. Later on I have to delete the temporary files created by pdftk ( a pdf tool ) I use to split them up and also show the s3 links in the response body since this is an API request.
The way I have structured this is as following:
Inside my Split module where the core business logic is:
filenames = []
s3_links = []
Enum.map(pages, fn(item) ->
split_filename = item
|> split(filename)
link = split_filename
|> FileHelper.result_file_bytes()
|> ManageS3.upload()
|> FileHelper.save_file(work_group_id, pass)
[filenames ++ split_filename, s3_links ++ link]
end)
|> transform()
{filenames, s3_links}
The important things are split_filename and link
This is what I'm getting when I call an IO.inspect in the transform() method:
[
["87cdcd73-5b27-4757-a472-78aaf6cc6864.pdf",
"Some_S3_LINK00"],
["0ab460ca-5019-4864-b0ff-343966c7d72a.pdf",
"Some_S3_LINK01"]
]
The structuring is [[filename, s3_link], [filename, s3_link]] whereas the desired outcome would be that of [ [list of all filenames], [list of s3 links].
If anybody can lend a hand I would be super grateful. Thanks in advance!
Sidenotes:
Assigning filenames = []; s3_links = [] in the very beginning makes zero sense. Enum.map already maps the input. What you need is probably Enum.reduce/3.
Don’t use the pipe |> operator when the pipe consists of the only call, it is considered an anti-pattern by Elixir core team.
Always start pipes with a term.
Solution:
Reduce the input into the result using Enum.reduce/3 directly to what you need.
pages
|> Enum.reduce([[], []], fn item, [files, links] ->
split_filename = split(item, filename)
link =
split_filename
|> FileHelper.result_file_bytes()
|> ManageS3.upload()
|> FileHelper.save_file(work_group_id, pass)
[[split_filename | files], [link | links]]
end)
|> Enum.map(&Enum.reverse/1)
|> IO.inspect(label: "Before transform")
|> transform()
You did not provide the input to test it, but I believe it should work.
Instead of working on lists of lists, you may want to consider using tuples with lists. Something like the following should work for you.
List.foldl(pages, {[], []}, fn(item, {filenames, links}) ->
filename = split(item, filename)
link =
file_name
|> FileHelper.result_file_bytes()
|> ManagerS3.upload()
|> FileHelper.save_file(work_group_id, pass)
{[filename | filenames], [link | links]}
end)
This will return a value that looks like
{
["87cdcd73-5b27-4757-a472-78aaf6cc6864.pdf",
"0ab460ca-5019-4864-b0ff-343966c7d72a.pdf"],
["Some_S3_LINK00",
"Some_S3_LINK01"]
}
Though, depending on how you are using these values, maybe a list of tuples would be more appropriate. Something like
Enum.map(pages, fn(item) ->
filename = split(item, filename)
link =
filename
|> FileHelper.result_file_bytes()
|> ManageS3.upload()
|> FileHelper.save_file(work_group_id, pass)
{filename, link}
end)
would return
[
{"87cdcd73-5b27-4757-a472-78aaf6cc6864.pdf", "Some_S3_LINK00"},
{"0ab460ca-5019-4864-b0ff-343966c7d72a.pdf", "Some_S3_LINK01"}
]
I'm writing a test to check a function (called automatically by GenServer when a new file enters a folder) that calls other functions in the same module with pipes in order to read a file, process its content to insert it if needed and returns a list (:errors and :ok maps).
results looks like :
[
error: "Data not found",
ok: %MyModule{
field1: field1data,
field2: field2data
},
ok: %MyModule{
field1: field1data,
field2: field2data
},
error: "Data not found"
the code :
def processFile(file) do
insertResultsMap =
File.read!(file)
|> getLines()
|> extractMainData()
|> Enum.map(fn(x) -> insertLines(x) end)
|> Enum.group_by(fn x -> elem(x, 0) end)
handleErrors(Map.get(insertResultsMap, :error))
updateAnotherTableWithLines(Map.get(insertResultsMap, :ok))
end
defp getLines(docContent) do
String.split(docContent, "\n")
end
defp extractMainData(docLines) do
Enum.map(fn(x) -> String.split(x, ",") end)
end
defp insertLines([field1, field2, field3, field4]) do
Attrs = %{
field1: String.trim(field1),
field2: String.trim(field2),
field3: String.trim(field3),
field4: String.trim(field4)
}
mymodule.create_stuff(Attrs)
end
defp handleErrors(errors) do
{:ok, file} = File.open(#errorsFile, [:append])
saveErrors(file, errors)
File.close(file)
end
defp saveErrors(_, []), do: :ok
defp saveErrors(file, [{:error, changeset}|rest]) do
changes = for {key, value} <- changeset.changes do
"#{key} #{value}"
end
errors = for {key, {message, _}} <- changeset.errors do
"#{key} #{message}"
end
errorData = "data: #{Enum.join(changes, ", ")} \nErrors: #{Enum.join(errors, ", ")}\n\n"
IO.binwrite(file, errorData)
saveErrors(file, rest)
end
defp updateAnotherTableWithLines(insertedLines) do
Enum.map(insertedLines, fn {:ok, x} -> updateOtherTable(x) end)
end
defp updateOtherTable(dataForUpdate) do
"CLOSE" -> otherModule.doStuff(dataForUpdate.field1, dataForUpdate.field2)
end
I have several questions, and some will be pretty basic since I'm still learning :
What do you think of the code ? Any advices ? (take into account I voluntarily obfuscated names).
If I want to test this, is it the right way to test only processFile function ? Or should I make public more of them and test them individually ?
When I test the processFile function, I check that I'm receiving a list. Any way to make sure this list has only elements I'm waiting for, thus error: "String" or ok: %{}" ?
What do you think of the code? Any advices? (take into account I voluntarily obfuscated names).
Opinion based.
If I want to test this, is it the right way to test only processFile function?
Yes.
Or should I make public more of them and test them individually?
No, this is an implementation detail and testing it is an anti-pattern.
When I test the processFile function, I check that I'm receiving a list. Any way to make sure this list has only elements I'm waiting for, thus error: "String" or ok: %{}"?
You receive a Keyword. To check the explicit value, one might use:
foo = processFile(file)
assert not is_nil(foo[:ok])
OTOH, I’d better return a map from there and pattern match it:
assert %{ok: _} = processFile(file)
To assert that the result does not have anything save for :oks and :errors, one might use list subtraction:
assert Enum.uniq(Keyword.keys(result)) -- [:ok, :error] == []
In the build systems that I'm familiar with (make and msbuild) there's a way to specify the inputs and outputs for a target. If the time stamps on the input files are earlier than those on the outputs the task is skipped. I can't find something similar in FAKE.
For example if I wanted to translate this Makefile to Fake
a.exe: a.fs
fsharpc a.fs -o a.exe
it might look like:
Target "a.exe" (fun _ -> ["a.fs"] |> FscHelper.compile [...])
However, when I run the build command it will always execute the compiler and produce a new a.exe regardless the modification time on a.fs. Is there a simple way to get the same behavior as the makefile?
You could use =?>and provide a function that returns true or false if the task should run.
let fileModified f1 f2 =
FileInfo(f1).LastWriteTime > FileInfo(f2).LastWriteTime
and then in target dependencies
=?> ("a.exe", fileModified "a.fs" "a.exe")
A more complete code example to flesh out Lazydevs answer:
#r "packages/FAKE/tools/FakeLib.dll"
open Fake
open System.IO
Target "build" (fun _ ->
trace "built"
)
let needsUpdate f1 f2 =
let lastWrite files =
files
|> Seq.map (fun f -> FileInfo(f).LastWriteTime)
|> Seq.max
let t1 = lastWrite f1
let t2 = lastWrite f2
t1 > t2
let BuildTarget name infiles outfiles fn =
Target name (fn infiles)
name =?> ("build", needsUpdate infiles outfiles)
BuildTarget "compile" ["Test2.fs"; "Test1.fs"] ["Test2.dll"] (fun files _ ->
files
|> FscHelper.compile [
FscHelper.Target FscHelper.TargetType.Library
]
|> function 0 -> () | c -> failwithf "compile error"
)
RunTargetOrDefault "build"
I'm trying to create a REST API using Phoenix with no Ecto or brunch.
What's the syntax for creating a post function in the router/controller with parameters, but not using Ecto?
For example in Ruby/Sinatra it would look something like this:
post "/v1/ipf" do
#weight1 = params[:weight1]
#weight2 = params[:weight2]
#weight3 = params[:weight3]
#weight4 = params[:weight4]
#goal1_percent = params[:goal1_percent]
#goal2_percent = params[:goal2_percent]
# etc...
end
Update
Based on Nick's answer, here's what I ended up with:
rest_api/web/router.ex:
defmodule RestApi.Router do
use RestApi.Web, :router
pipeline :api do
plug :accepts, ["json"]
end
scope "/", RestApi do
pipe_through :api
scope "/v1", V1, as: :v1 do
get "/ipf", IPFController, :ipf
end
end
end
rest_api/web/controllers/v1/ipf_controller.ex:
defmodule RestApi.V1.IPFController do
use RestApi.Web, :controller
import IPF
def ipf(conn, params) do
{weight1, _} = Integer.parse(params["weight1"])
{weight2, _} = Integer.parse(params["weight2"])
{weight3, _} = Integer.parse(params["weight3"])
{weight4, _} = Integer.parse(params["weight4"])
{goal1_percent, _} = Float.parse(params["goal1_percent"])
{goal2_percent, _} = Float.parse(params["goal2_percent"])
results = IPF.ipf(weight1, weight2, weight3, weight4, goal1_percent, goal2_percent)
render conn, results: results
end
end
rest_api/web/views/v1/ipf_view.ex:
defmodule RestApi.V1.IPFView do
use RestApi.Web, :view
def render("ipf.json", %{results: results}) do
results
end
end
Ecto and Brunch don't really have anything to do w/Phoenix handling a POST. Brunch is a web asset build tool, and Ecto is a database layer.
To add this new route, you just need to add an entry in the router for the new route:
post "/v1/spf", SPFController, :spf
And then create the controller:
defmodule MyModule.SPFController do
def spf(conn, params) do
# do whatever
end
end
That's it.
So, I have this ruby file and I want to use it in my rails project, but I don't have a clue on where or how to start, I've read about include and require, some sites tell me to use require, some to use include, and even to use both, but that's it. I would like to know where to put the necessary code, how to use the file's methods and where to put the file in the project directory, because I want to call it from a view but I don't know if that is the best, I'm sorry if this is a dumb question but rails is still new for me. I appreciate all the help you can offer. Thanks for your time.
The file I'm trying to use is the one that converts numbers to words created by Faustino Vasquez limon and it's a full class:
numeros.rb
class Numlet
def initialize(numero)
#numero = numero.to_s.reverse.split("")
#i = 0
#j = 0
#parte1 = []
#parte2 = []
#especial = ""
#numlet = []
#bandera=0
#bandera1=0
#a =[["Uno","Dos","Tres","Cuatro","Cinco","Seis","Siete","Ocho","Nueve"],
["Diez","Veinte","Treinta","Cuarenta","Cincuenta","Sesenta","Setenta","Ochenta","Noventa"],
["Ciento","Doscientos","Trescientos","Cuatrocientos","Quinientos","Seiscientos","Setecientos","Ochocientos","Novecientos"]]
end
def especial
#numlet[#j] = case #especial
when "11"then "Once"
when "12"then "Doce"
when "13"then "Trece"
when "14"then "Catorce"
when "15"then "Quice"
when "16"then "Dieciseis"
when "17"then "Diecisiete"
when "18"then "Dieciocho"
when "19"then "Diecinueve"
when "21"then "Veintiun"
when "22"then "Veintidos"
when "23"then "Veintitres"
when "24"then "Veinticuatro"
when "25"then "Veinticinco"
when "26"then "Veintiseis"
when "27"then "Veintisite"
when "28"then "Veintiocho"
when "29"then "Veintinueve"
else return 0
end
end
def repetir
case #numero.length
when 0..3 then #parte1[0] = #numero[0..#numero.length]
when 4..6 then #parte1[0] = #numero[0..2];#parte1[1] = #numero[3..#numero.length]
when 7..9 then #parte1[0] = #numero[0..2];#parte1[1] = #numero[3..5]; #parte1[2] = #numero[6..#numero.length]
else return 0
end
end
def convierte
#bandera1=0
#i=0
case #bandera
when 1 then #numlet[#j]="mil";#j+=1
when 2 then (#parte2.length==1 and #parte2[0]==1) ? #numlet[#j]="millon" : #numlet[#j]="millones";#j+=1
end
#especial = [#parte2[#i+1],#parte2[#i]].to_s
if especial != 0
#i+=2
#j+=1
else
if #parte2[#i].to_s =="1"
#numlet[#j]="Un"
#i+=1
#j+=1
end
end
while #i < #parte2.length
if #parte2[#i].to_i ==0
#i+=1
#bandera1+=1
else
if #parte2.length != 1 and #bandera1 ==0
if #i == 1
#numlet[#j]="y"
#j+=1
end
end
#numlet[#j] = #a[#i][#parte2[#i].to_i-1]
if #i == 2 and #bandera1==2 and #numlet[#j]=="Ciento"
#numlet[#j]="Cien"
end
#j+=1
#i+=1
end
end
#bandera+=1
end
def termina
#numlet.reverse.join(" ")
end
def a_letra
if repetir != 0
#parte1.each do |#parte2|
convierte
end
print "#{termina}\n"
else
print "Este numero no puede ser convertido\n"
end
end
end
That is what i want to use from my app. Thank you for your time.
Since it's your own class and you want to use it in one of the Rails views, there's many ways of using that code.
My own way and you may or may not choose to do it this way would be to include that file in the lib directory and then require it in the helper file of the you view you want to include it in.
This would allow you to access methods and initializers from that file and you would be able to create methods in your helpers to use on your views.