I'm having problems with one of my servers. I have simple app that reads files from ftp and stores in database.
After maintenance restart application is crashing after opening FTP connection.CPU rises til process is killed by oom.
Slogan: eheap_alloc: Cannot allocate 255489152 bytes of memory (of type "heap").
System version: Erlang/OTP 23 [erts-11.0] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [hipe]
Compiled: Tue Aug 4 13:49:01 2020
Taints: crypto
Atoms: 29346
Calling Thread: scheduler:0
=scheduler:1
Scheduler Sleep Info Flags: SLEEPING | TSE_SLEEPING | WAITING
Scheduler Sleep Info Aux Work: THR_PRGR_LATER_OP
Current Port:
Run Queue Max Length: 0
Run Queue High Length: 0
Run Queue Normal Length: 0
Run Queue Low Length: 0
Run Queue Port Length: 0
Run Queue Flags: OUT_OF_WORK | HALFTIME_OUT_OF_WORK
Current Process:
=scheduler:2
Scheduler Sleep Info Flags: SLEEPING | POLL_SLEEPING | WAITING
Scheduler Sleep Info Aux Work:
Current Port:
Run Queue Max Length: 0
Run Queue High Length: 0
Run Queue Normal Length: 0
Run Queue Low Length: 0
Run Queue Port Length: 0
Run Queue Flags: OUT_OF_WORK | INACTIVE
Current Process:
=dirty_cpu_scheduler:3
Scheduler Sleep Info Flags: SLEEPING | TSE_SLEEPING | WAITING
Scheduler Sleep Info Aux Work:
Current Process:
=dirty_cpu_scheduler:4
Scheduler Sleep Info Flags:
Scheduler Sleep Info Aux Work:
Current Process: <0.1121.0>
Current Process State: Garbing
Current Process Internal State: ACT_PRIO_NORMAL | USR_PRIO_NORMAL | PRQ_PRIO_NORMAL | ACTIVE | GC | DIRTY_ACTIVE_SYS | DIRTY_RUNNING_SYS
Current Process Program counter: 0x00007f5941057438 ('Elixir.Ecto.Adapters.Postgres.TypeModule':'Elixir.Ecto.Adapters.Postgres.Timestamp'/6 + 240)
Current Process Limited Stack Trace:
0x00007f594144f290:SReturn addr 0x436E35F8 ('Elixir.Postgrex.Protocol':rows_recv/4 + 136)
0x00007f594144f2a8:SReturn addr 0x436D8740 ('Elixir.Postgrex.Protocol':execute_recv/5 + 232)
0x00007f594144f2d8:SReturn addr 0x43688A80 ('Elixir.DBConnection':handle/4 + 400)
0x00007f594144f320:SReturn addr 0x43687928 ('Elixir.DBConnection':describe_run/5 + 360)
0x00007f594144f360:SReturn addr 0x4368DA80 ('Elixir.DBConnection':'-run_meter/5-fun-0-'/4 + 136)
0x00007f594144f388:SReturn addr 0x4368B888 ('Elixir.DBConnection':run_begin/3 + 112)
0x00007f594144f3b8:SReturn addr 0x4368A628 ('Elixir.DBConnection':prepare_execute/4 + 144)
0x00007f594144f3d8:SReturn addr 0x4367E5B0 ('Elixir.Ecto.Adapters.Postgres.Connection':prepare_execute/5 + 192)
0x00007f594144f3e0:SReturn addr 0x80320E18 ('Elixir.Ecto.Adapters.SQL':sql_call/6 + 840)
0x00007f594144f420:SReturn addr 0x8031ED38 ('Elixir.Ecto.Adapters.SQL':execute_and_cache/7 + 408)
0x00007f594144f460:SReturn addr 0x410E3FC8 ('Elixir.Ecto.Repo.Queryable':execute/5 + 864)
0x00007f594144f4c8:SReturn addr 0x410E37E0 ('Elixir.Ecto.Repo.Queryable':all/4 + 240)
0x00007f594144f4d0:SReturn addr 0x803009E0 ('Elixir.Neptun.Fetchers.FTPFetcher':filter_listing_for_downloaded_data/1 + 336)
0x00007f594144f4e8:SReturn addr 0x803002F0 ('Elixir.Neptun.Fetchers.FTPFetcher':do_work/0 + 248)
0x00007f594144f508:SReturn addr 0x803012E0 ('Elixir.Neptun.Fetchers.FTPFetcher':handle_info/2 + 504)
0x00007f594144f520:SReturn addr 0x84612198 (gen_server:try_dispatch/4 + 152)
0x00007f594144f568:SReturn addr 0x84613038 (gen_server:handle_msg/6 + 1696)
0x00007f594144f5a0:SReturn addr 0x8461B840 (proc_lib:init_p_do_apply/3 + 72)
0x00007f594144f5c0:SReturn addr 0x6A011F08 (<terminate normally="" process="">)
=dirty_cpu_run_queue
Run Queue Max Length: 0
Run Queue High Length: 0
Run Queue Normal Length: 0
Run Queue Low Length: 0
Run Queue Port Length: 0
Run Queue Flags: OUT_OF_WORK | HALFTIME_OUT_OF_WORK | NONEMPTY | EXEC
=dirty_io_scheduler:5
Scheduler Sleep Info Flags: SLEEPING | TSE_SLEEPING | WAITING
Scheduler Sleep Info Aux Work:
Current Process:
=dirty_io_scheduler:6</terminate>
elixir version: 1.7.4-otp-21
erlang/otp: 21.3
postgresql: 9.6
I tried compiling with newer versions with no result.
A appreciate your feedback. Thank you.
edit: When opening ftp connection app should read filename in format 0000000401_20191110120118, open the file and write information from file to db. File contains data in this format:
0000000401;0001;20191110;120000;11.20
0000000401;0002;20191110;120000;7.82
defmodule Neptun.FTP do
#host 'xx.xx.xx.xx'
#username 'xx'
#password 'xxx'
#dir '/xx'
#sep "\r\n"
require Logger
def ls() do
case authenticated_client() do
{:ok, pid} ->
{:ok, listing} = :ftp.ls(pid, #dir)
listing =
listing
|> List.to_string
|> parse_listing
:ok = close_client(pid)
{:ok, listing}
{:error, msg} ->
{:error, msg}
end
end
def get_file(filename) do
case authenticated_client() do
{:ok, pid} ->
case get_file(pid, filename) do
{:ok, file} ->
:ok = close_client(pid)
{:ok, file}
{:error, msg} ->
{:error, msg}
end
{:error, msg} ->
{:error, msg}
end
end
def get_file(pid, filename) do
result = :ftp.recv_bin(pid, #dir ++ '/' ++ String.to_charlist(filename))
result
end
def get_files(filenames) do
case authenticated_client() do
{:ok, pid} ->
files = filenames
|> Enum.map(fn filename ->
get_file(pid, filename)
end)
:ok = close_client(pid)
files
{:error, msg} ->
msg
end
end
def authenticated_client do
with {:ok, pid} <- :ftp.open(#host),
:ok <- :ftp.user(pid, #username, #password) do
IO.puts "Opened FTP connection"
IO.inspect pid
{:ok, pid}
else
_ -> {:error, "Unable to authenticate"}
end
end
def close_client(pid) do
IO.puts "Closed FTP connection"
IO.inspect pid
:ftp.close(pid)
end
defp parse_listing(raw_listing) do
raw_listing
|> String.split(#sep)
|> Enum.map(fn line ->
case String.split(line) do
[_perm, _num, _user, _group, _size, _month, _day, _time, filename] ->
filename
_ ->
""
end
end)
|> Enum.filter(fn line ->
len = line |> String.trim() |> String.length
len > 0
end)
end
end
FTPFetcher:
defmodule Neptun.Fetchers.FTPFetcher do
use GenServer
alias Neptun.Repo
# API
def start_link(name \\ FTPFetcher) do
# Instead of passing an atom to the `name` option, we send
# a tuple. Here we extract this tuple to a private method
# called `via_tuple` that can be reused for every function
GenServer.start_link(__MODULE__, %{name: name}, name: via_tuple(name))
end
def init(state) do
allow_work()
{:ok, state}
end
defp via_tuple(device) do
# And the tuple always follow the same format:
# {:via, module_name, term}
{:via, :gproc, {:n, :l, {:device, device}}}
end
def handle_info(:work, %{name: name}=state) do
IO.puts "Getting data for #{name}..."
{:ok, log} = Neptun.Logs.ftp_fetching_started
:ok = do_work()
{:ok, _log} = Neptun.Logs.job_finished(log)
allow_work()
{:noreply, state, :hibernate}
end
defp filter_listing_for_valid_stations(listing) do
IO.puts "Filtering listing of length #{length(listing)} for unknown stations"
station_uids = Neptun.Stations.list_stations |> Enum.map(fn station -> station.uid end)
IO.puts "Existing stations: #{length(station_uids)}"
filtered_listing = listing
|> Enum.filter(fn l ->
case Neptun.Utils.parse_ftp_filename(l) do
[station_uid, _datetime] ->
Enum.member?(station_uids, station_uid)
nil ->
IO.puts "Skipping line #{l}"
false
end
end)
IO.puts "Filtered listing length #{length(filtered_listing)}"
filtered_listing
end
def filter_listing_for_downloaded_data(listing) do
IO.puts "Filtering listing of length #{length(listing)} for downloaded data"
data_filenames = Neptun.Stations.list_data |> Enum.map(fn data -> data.filename end)
IO.puts "Existing data items: #{length(data_filenames)}"
filtered_listing = MapSet.difference(MapSet.new(listing), MapSet.new(data_filenames)) |> MapSet.to_list
IO.puts "Filtered listing length #{length(filtered_listing)}"
filtered_listing
end
defp do_work() do
listing = Neptun.FTP.ls
listing
|> filter_listing_for_valid_stations()
|> filter_listing_for_downloaded_data()
|> Enum.each(fn l ->
try do
Repo.transaction(fn ->
case Neptun.Stations.get_data_by_filename(l) do
nil ->
store_data(l)
_ ->
nil
#IO.puts "#{l} already exists, skipping"
end
end)
rescue
e ->
store_data_error(l, e)
end
end)
Neptun.Processors.DataProcessor.process_data()
:ok
end
defp store_data_error(filename, e) do
IO.puts "Error while saving #{filename}"
[station_uid, _datetime] = Neptun.Utils.parse_ftp_filename(filename)
station = Neptun.Stations.get_station_by_uid(station_uid)
params = %{
"station_id" => station.id,
"content" => "",
"processed" => :false,
"filename" => filename,
"status" => Neptun.Stations.DataStatus.error,
"message" => Exception.message(e)
}
{:ok, _data} = Neptun.Stations.create_data(params)
end
defp store_data(l) do
IO.puts "Data #{l} not found, getting from server"
{:ok, file} = Neptun.FTP.get_file(l)
IO.puts "Saving #{l}"
status = if file == "" do
IO.puts "File #{l} is empty"
Neptun.Stations.DataStatus.empty
else
Neptun.Stations.DataStatus.pending
end
[station_uid, datetime] = Neptun.Utils.parse_ftp_filename(l)
case Neptun.Stations.get_station_by_uid(station_uid) do
nil ->
IO.puts "Station with UID #{station_uid} does not exist"
nil
station ->
IO.puts "Station with UID #{station_uid} found"
dt =
Neptun.Utils.parse_csv_datetime(datetime)
|> Neptun.Utils.set_timezone(station.timezone)
params = %{
"station_id" => station.id,
"content" => to_string(file),
"processed" => :false,
"filename" => l,
"status" => status,
}
{:ok, _data} = Neptun.Stations.create_data(params)
params = %{
"last_seen" => dt
}
IO.puts "PARAMS for station"
IO.inspect params
{:ok, _station} = Neptun.Stations.update_station(station, params)
end
end
defp allow_work() do
timeout = Application.get_env(:neptun, :ftp_fetcher_timeout)
Process.send_after(self(), :work, timeout)
end
end
See #Aleksei Matiushkin's comment about limiting the result of Repo.all
Related
There is a program of three modules. The Print module receives a number from the keyboard, passes it to another module, receives the response, and displays it on the screen. The Proc1 and Proc2 modules receive a number, perform calculations, and send the result back.
defmodule Launch do
#moduledoc """
Documentation for `Launch`.
"""
#doc """
"""
def start() do
children = [
%{
id: Print,
start: {Print, :print, []}
},
%{
id: Proc1,
start: {Proc1, :proc1, []}
},
%{
id: Proc2,
start: {Proc2, :proc2, []}
}
]
Supervisor.start_link(children, strategy: :one_for_one)
end
end
defmodule Print do
def print() do
num =
IO.gets("Input number: ")
|> String.trim()
|> String.to_integer()
if num >= 0 do
send(Proc1, {self(), num})
else
send(Proc2, {self(), num})
end
receive do
num -> IO.puts(num)
after
500 ->
print()
end
print()
end
end
defmodule Proc1 do
def proc1() do
receive do
{pid, num} ->
send(pid, 100/num)
proc1()
_e ->
IO.puts("Error")
end
end
end
defmodule Proc2 do
def proc2() do
receive do
{pid, num} ->
send(pid, 1000/num)
proc2()
_e ->
IO.puts("Error")
end
end
end
I am trying to run all processes under the supervision of a single Supervisor. But there is a problem-only the first "child" is started, the other "children" are not started. In the example above, the Print process will start, but Proc1 and Proc2 will not start. How do I run all processes under one Supervisor? Important note: the Print process must get the addresses of the Proc1 and Proc2 processes for communication.
There are many issues with the code you’ve posted.
Registered processes
To be able to use process name as Process.dest() in a call to Kernel.send/2, one should start the named process.
Supervisor.start_link/2
Supervisor.start_link/2 expects a list of tuples, with modules and functions that immediately return, having the process started as a side effect. These functions are called, and there would not be any magic: if this is an infinitely recursive function, the execution flow would be deadlocked inside, waiting for the message in receive/1.
Supervisor performs some magic by automatically monitoring and restarting children for you, but it does nothing to spawn the separate processes. GenServer encapsulates this functionality and provides a handy way to not bother about spawning processes.
Solution
What you might do, is to spawn all three processes, manually monitor them, and react on {:DOWN, ref, :process, pid, reason} message respawning the died process. This is exactly what Supervisor effectively does under the hood for children.
Launch
defmodule Launch do
def start() do
proc1 = spawn(&Proc1.proc1/0)
proc2 = spawn(&Proc2.proc2/0)
print = spawn(fn -> Print.print(proc1, proc2) end)
Process.monitor(proc1)
Process.monitor(proc2)
Process.monitor(print)
receive do
msg -> IO.inspect(msg)
end
end
end
Print
defmodule Print do
def print(pid1, pid2) do
num =
IO.gets("Input number: ")
|> String.trim()
|> String.to_integer()
if num >= 0 do
send(pid1, {self(), num})
else
send(pid2, {self(), num})
end
receive do
num -> IO.puts(num)
end
print(pid1, pid2)
end
end
The other two modules are fine.
Here is how it will look like in iex
iex|1 ▶ c "/tmp/test.ex"
#⇒ [Launch, Print, Proc1, Proc2]
iex|2 ▶ Launch.start
Input number: 10
10.0
Input number: 1000
0.1
Input number: a
#⇒ {:DOWN, #Reference<0.3632020665.3980394506.95298>,
# :process, #PID<0.137.0>,
# {:badarg,
# [
# {:erlang, :binary_to_integer, ["a"], []},
# {Print, :print, 2, [file: '/tmp/test.ex', line: 22]}
# ]}}
Now instead of printing this out, respawn the failed process, and you will get a bare implementation of the supervised intercommunicating processes. For all_for_one strategy that could be achieved with:
receive do
{:DOWN, _, _, _, _} ->
Process.exit(print, :normal)
Process.exit(proc1, :normal)
Process.exit(proc2, :normal)
start()
end
In my application there is a GenServer, which can create other processes. All process IDs are saved to a list.
def create_process do
GenServer.call(__MODULE__, :create_process)
end
def handle_call(:create_process, _from, processes) do
{:ok, pid} = SomeProcess.start_link([])
{:reply, {:ok, pid}, [pid | processes]}
end
There is also a function to get the list of PIDs.
def get_processes do
GenServer.call(__MODULE__, :get_processes)
end
def handle_call(:get_processes, _from, processes) do
{:reply, processes, processes}
end
I tried to write a doctest for the get_processes function like this:
#doc """
iex> {:ok, pid} = MainProcess.create_process()
iex> MainProcess.get_processes()
[pid]
"""
However the test runner doesn't seem to see the pid variable, and I get an undefined function pid/0 error.
I know it could be simply solved in with a regular test, but i want to know it it possible to solve in doctest.
The problem is the [pid] line in your expected result. The expected result should be an exact value, not a variable. You can't reference the variable from the expected result. You can work around it by checking the pid on the previous line:
iex> {:ok, pid} = MainProcess.create_process()
iex> [pid] === MainProcess.get_processes()
true
I want to test that messages that are broadcast when some background jobs are completed are actually appearing in the view.
I have unit tests for this which work fine. I would actually like to ensure that the JS gets run so that the view is updated with the correct message.
So far I have not been able to find any way to do this.
Here is the test I have where I would like to add the expectation for the broadcast message:
require 'rails_helper'
require 'sidekiq/testing'
RSpec.describe 'sending a quote request', js: true do
let(:quote_request_form) { build(:quote_request_form) }
before do
create(:job_rate, :proofreading)
create(:proofreader_with_work_events)
end
it 'shows the user their quotation' do
visit new_quote_request_path
fill_in("quote_request_form_name", with: quote_request_form.name)
fill_in("quote_request_form_email", with: quote_request_form.email)
attach_file('customFile','/Users/mitchellgould/RailsProjects/ProvenWordNew/spec/test_documents/quote_request_form/1.docx', make_visible: true)
click_on "Submit"
Sidekiq::Testing.inline! do
page.execute_script("$('#invisible-recaptcha-form').submit()")
expect(current_path).to eq(quote_confirm_path)
#add expectation here:
expect(page).to have_content("Calculating Time Required")
page.execute_script("window.location.pathname = '#{quotation_path(Quotation.first)}'")
expect(current_path).to eq(quotation_path(Quotation.first))
expect(page).to have_content("Here is your quotation")
end
end
end
Here is my .coffee file:
$(document).on 'turbolinks:load', ->
if $("meta[name='current_user']").length > 0
App.notification = App.cable.subscriptions.create "NotificationChannel",
connected: ->
# Called when the subscription is ready for use on the server
disconnected: ->
# Called when the subscription has been terminated by the server
received: (data) ->
$('.background_message').html(data.content)
if data.head == 302 && data.path
window.location.pathname = data.path
else if App.notification
App.quotation.unsubscribe()
delete App.notification
Here is one of the background jobs that broadcasts a message when its done:
class CreateParagraphDetailsJob < ApplicationJob
queue_as :default
after_perform :broadcast_message, :calculate_proofreading_job_duration
def perform(document, proofreading_job_id, current_user_id)
document.create_paragraph_details
end
private
def calculate_proofreading_job_duration
CalculateDurationJob.set(wait: 1.seconds).perform_later proofreading_job_id, current_user_id
end
def broadcast_message
ActionCable.server.broadcast "notification_channel_user_#{current_user_id}", content: "Analyzed writing quality of paragraphs"
end
def document
self.arguments.first
end
def proofreading_job_id
self.arguments.second
end
def current_user_id
self.arguments.last
end
end
Any ideas on how to do this?
I'm writing a test to check a function (called automatically by GenServer when a new file enters a folder) that calls other functions in the same module with pipes in order to read a file, process its content to insert it if needed and returns a list (:errors and :ok maps).
results looks like :
[
error: "Data not found",
ok: %MyModule{
field1: field1data,
field2: field2data
},
ok: %MyModule{
field1: field1data,
field2: field2data
},
error: "Data not found"
the code :
def processFile(file) do
insertResultsMap =
File.read!(file)
|> getLines()
|> extractMainData()
|> Enum.map(fn(x) -> insertLines(x) end)
|> Enum.group_by(fn x -> elem(x, 0) end)
handleErrors(Map.get(insertResultsMap, :error))
updateAnotherTableWithLines(Map.get(insertResultsMap, :ok))
end
defp getLines(docContent) do
String.split(docContent, "\n")
end
defp extractMainData(docLines) do
Enum.map(fn(x) -> String.split(x, ",") end)
end
defp insertLines([field1, field2, field3, field4]) do
Attrs = %{
field1: String.trim(field1),
field2: String.trim(field2),
field3: String.trim(field3),
field4: String.trim(field4)
}
mymodule.create_stuff(Attrs)
end
defp handleErrors(errors) do
{:ok, file} = File.open(#errorsFile, [:append])
saveErrors(file, errors)
File.close(file)
end
defp saveErrors(_, []), do: :ok
defp saveErrors(file, [{:error, changeset}|rest]) do
changes = for {key, value} <- changeset.changes do
"#{key} #{value}"
end
errors = for {key, {message, _}} <- changeset.errors do
"#{key} #{message}"
end
errorData = "data: #{Enum.join(changes, ", ")} \nErrors: #{Enum.join(errors, ", ")}\n\n"
IO.binwrite(file, errorData)
saveErrors(file, rest)
end
defp updateAnotherTableWithLines(insertedLines) do
Enum.map(insertedLines, fn {:ok, x} -> updateOtherTable(x) end)
end
defp updateOtherTable(dataForUpdate) do
"CLOSE" -> otherModule.doStuff(dataForUpdate.field1, dataForUpdate.field2)
end
I have several questions, and some will be pretty basic since I'm still learning :
What do you think of the code ? Any advices ? (take into account I voluntarily obfuscated names).
If I want to test this, is it the right way to test only processFile function ? Or should I make public more of them and test them individually ?
When I test the processFile function, I check that I'm receiving a list. Any way to make sure this list has only elements I'm waiting for, thus error: "String" or ok: %{}" ?
What do you think of the code? Any advices? (take into account I voluntarily obfuscated names).
Opinion based.
If I want to test this, is it the right way to test only processFile function?
Yes.
Or should I make public more of them and test them individually?
No, this is an implementation detail and testing it is an anti-pattern.
When I test the processFile function, I check that I'm receiving a list. Any way to make sure this list has only elements I'm waiting for, thus error: "String" or ok: %{}"?
You receive a Keyword. To check the explicit value, one might use:
foo = processFile(file)
assert not is_nil(foo[:ok])
OTOH, I’d better return a map from there and pattern match it:
assert %{ok: _} = processFile(file)
To assert that the result does not have anything save for :oks and :errors, one might use list subtraction:
assert Enum.uniq(Keyword.keys(result)) -- [:ok, :error] == []
What I am trying (Discord Bot) is to make a command called !say serverID channelID arg
The point of this command is that I will pm the bot with this command and the bot writes the arg in the defined channelID from the server.
Library (litcord): https://github.com/satom99/litcord
A part of the code (Yes, this has issues...):
local server = client.servers:getAll('id', serverID)
if server then
if server.contains('id', serverID) then
for _, serverID in pairs(server) do
if serverID == serverID then
return true
end
return false
end
end
print(serverID, channelID, arg)
return end
The full code:
client:on(
'message',
function(message)
local userID = message.author.id
local cmd, serverID, channelID, arg = string.match(message.content, '(%S+) (%d+) (%d+) (%S+.*)')
local cmd = cmd or message.content
if (cmd == "!say") and message.parent.is_private then
if (userID == "187590758360940545") then
local server = client.servers:getAll('id', serverID)
if server then
if server.contains('id', serverID) then
for _, serverID in pairs(server) do
if serverID == serverID then
return true
end
return false
end
end
print(serverID, channelID, arg)
return end
local channel = server.channels:get('id', channelID)
client.channel = channelID
message.channel:sendMessage(arg)
else
message:reply(":sob: Stop!!!!")
return end
end
end
)
Things I am trying to:
Check the table to see if it has a server
and the same for channel
etc....