Django: object needs to have a value for field "..." before this many-to-many relationship can be used - sql

I experience a strange error with Django 1.5:
I have defined a model like below:
class Company(models.Model):
user = models.OnetoOneField(User)
agreed_to_terms = models.NullBooleanField(default=False)
address = models.CharField(_('Complete Address'),
max_length = 255, null = True, blank = True)
winning_bid = models.ForeignKey('Bid',
related_name='winning_bid',
blank = True, null = True)
bid_list = models.ManyToManyField('Bid',
related_name='bids',
blank = True, null = True)
...
class Bid(models.Model):
user = models.ForeignKey(User, null = True, blank = True)
description = models.TextField(_('Description'),
blank = True, null = True,)
volume = models.DecimalField(max_digits=7, decimal_places=3,
null=True, blank=True,)
...
# all other attributes are of the Boolean, CharField or DecimalField type. No Foreignkeys, nor ManytoManyFields.
When I try to file the form with the initial data through the Django admin, I get the following error:
Exception Value:
"" needs to have a value for field "company" before this many-to-many relationship can be used.
Please see the traceback below.
The error message does not make very much sense to me. The only m2m relationship is bid_list, which is null = True and was null at the time of saving.
Is there something new in Django 1.5, which I have not discovered while reading the changelog (this is my first project in Django 1.5)?
Interestingly, when I save an object in the Django shell, I do not get an error message, but the object does not get saved without any error message.
In [1]: user = User.objects.get(username='admin')
In [2]: new_company = Company()
In [3]: new_company.user = user
In [4]: new_company.save() Out[4]: <Company: Company object>
In [5]: foo = Company.objects.all()
Out[5]: []
When I try to trace the SQL statements with the debug toolbar, I can only see SQL SELECT statements, no INSERT requests.
What is the explanation for this strange behaviour?
Traceback:
Request Method: POST
Request URL: /admin/company/company/add/
Django Version: 1.5.1
Python Version: 2.7.1
Installed Applications:
('django.contrib.admin',
'django.contrib.admindocs',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.gis',
'django.contrib.humanize',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.admin',
'django.contrib.admindocs',
'crispy_forms',
'django_extensions',
'easy_thumbnails',
'registration',
'south',
'company',
'bid',
'debug_toolbar')
Installed Middleware:
('django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'debug_toolbar.middleware.DebugToolbarMiddleware')
Traceback:
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/core/handlers/base.py" in get_response
115. response = callback(request, *callback_args, **callback_kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/contrib/admin/options.py" in wrapper
372. return self.admin_site.admin_view(view)(*args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/utils/decorators.py" in _wrapped_view
91. response = view_func(request, *args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/views/decorators/cache.py" in _wrapped_view_func
89. response = view_func(request, *args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/contrib/admin/sites.py" in inner
202. return view(request, *args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/utils/decorators.py" in _wrapper
25. return bound_func(*args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/utils/decorators.py" in _wrapped_view
91. response = view_func(request, *args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/utils/decorators.py" in bound_func
21. return func(self, *args2, **kwargs2)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/db/transaction.py" in inner
223. return func(*args, **kwargs)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/contrib/admin/options.py" in add_view
1008. self.save_related(request, form, formsets, False)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/contrib/admin/options.py" in save_related
762. form.save_m2m()
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/forms/models.py" in save_m2m
84. f.save_form_data(instance, cleaned_data[f.name])
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/db/models/fields/related.py" in save_form_data
1336. setattr(instance, self.attname, data)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/db/models/fields/related.py" in __set__
910. manager = self.__get__(instance)
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/db/models/fields/related.py" in __get__
897. through=self.field.rel.through,
File "/Users/neurix/Development/virtual/lib/python2.7/site-packages/django/db/models/fields/related.py" in __init__
586. (instance, source_field_name))
Exception Type: ValueError at /admin/company/company/add/
Exception Value: "<Company: Company object>" needs to have a value for field "company" before this many-to-many relationship can be used.
settings.py
import os, os.path, sys
DEBUG = True
TEMPLATE_DEBUG = DEBUG
# Setting up folders
abspath = lambda *p: os.path.abspath(os.path.join(*p))
PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
TASK2_MODULE_PATH = abspath(PROJECT_ROOT, 'apps/')
sys.path.insert(0, TASK2_MODULE_PATH)
# Loading passwords
try:
from settings_pwd import *
except ImportError:
pass
AUTH_PROFILE_MODULE = 'profile.UserProfile'
#ALLOWED_HOSTS = [''] # not needed for DEBUG = True
TIME_ZONE = 'Europe/London'
LANGUAGE_CODE = 'en-uk'
LANGUAGES = [
("en", u"English"),
]
SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = True
if DEBUG:
MEDIA_ROOT = os.path.join(PROJECT_ROOT, "site_media", "media")
else:
MEDIA_ROOT = "folder_to_upload_files"
if DEBUG:
MEDIA_URL = "/media/"
else:
MEDIA_URL = "/media/uploads/"
if DEBUG:
STATIC_ROOT = os.path.join(PROJECT_ROOT, "site_media","static")
else:
STATIC_ROOT = "folder_to_static_files"
STATIC_URL = '/static/'
STATICFILES_DIRS = (
os.path.join(PROJECT_ROOT, "assets"),
)
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)
SECRET_KEY = '...'
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',)
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',)
ROOT_URLCONF = 'task2.urls'
WSGI_APPLICATION = 'task2.wsgi.application'
TEMPLATE_DIRS = (
os.path.join(PROJECT_ROOT, "templates"),
os.path.join(PROJECT_ROOT, "templates/pages"),)
INSTALLED_APPS = (
# Django apps
'django.contrib.admin',
'django.contrib.admindocs',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.gis',
'django.contrib.humanize',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.admin',
'django.contrib.admindocs',
# third party apps
'crispy_forms',
'django_extensions',
'easy_thumbnails',
'registration',
'south',
# task2 apps
'profile',
'company',
)
AUTHENTICATION_BACKENDS = (
'django.contrib.auth.backends.ModelBackend',
)
log = DEBUG
if log:
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'simple': {
'format': '%(levelname)s %(message)s',
},
},
'handlers': {
'console':{
'level':'DEBUG',
'class':'logging.StreamHandler',
'formatter': 'simple'
},
},
'loggers': {
'django': {
'handlers': ['console'],
'level': 'DEBUG',
},
}
}
####################
# THIRD PARTY SETUPS
# For Crispy Forms
CRISPY_FAIL_SILENTLY = not DEBUG
CRISPY_TEMPLATE_PACK = 'bootstrap'
## For Django Registration
ACCOUNT_ACTIVATION_DAYS = 7
# for Django testing to avoid conflicts with South migrations
SOUTH_TESTS_MIGRATE = False
# Debug_toolbar settings
if DEBUG:
INTERNAL_IPS = ('127.0.0.1',)
MIDDLEWARE_CLASSES += (
'debug_toolbar.middleware.DebugToolbarMiddleware',
)
INSTALLED_APPS += (
'debug_toolbar',
)
DEBUG_TOOLBAR_PANELS = (
'debug_toolbar.panels.version.VersionDebugPanel',
'debug_toolbar.panels.timer.TimerDebugPanel',
'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
'debug_toolbar.panels.headers.HeaderDebugPanel',
#'debug_toolbar.panels.profiling.ProfilingDebugPanel',
'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
'debug_toolbar.panels.sql.SQLDebugPanel',
'debug_toolbar.panels.template.TemplateDebugPanel',
'debug_toolbar.panels.cache.CacheDebugPanel',
'debug_toolbar.panels.signals.SignalDebugPanel',
'debug_toolbar.panels.logger.LoggingPanel',
)
DEBUG_TOOLBAR_CONFIG = {
'INTERCEPT_REDIRECTS': False,
}
# Easy_Thumbnail setup
THUMBNAIL_ALIASES = {
'': {
'thumbnail': {'size': (50, 50), 'crop': True},
},
}

The problem has to do with how you use your views.
I think you are using:
instance.add(many_to_many_instance)
before you have an instance id.
so first save your model:
instance.save()
instance.add(many_to_many_instance)

You are using custom user model AUTH_PROFILE_MODULE = 'profile.UserProfile', but in code I suppose you use native django user.
I suppose your models should be like
class Company(models.Model):
user = models.OnetoOneField('profile.UserProfile')
...
read more https://docs.djangoproject.com/en/1.5/ref/contrib/auth/#django.contrib.auth.models.User.get_profile

Related

Why am I getting 'Key Error' at for my file I am doing

Receiving a key error:
KeyError at /
'entries'
Request Method: GET
Request URL: http://localhost:8000/
Django Version: 3.2.14
Exception Type: KeyError
Exception Value:
'entries'
Exception Location: /home/project/agfzb-CloudAppDevelopment_Capstone/server/djangoapp/restapis.py, line 50, in get_all_dealerships_from_cf
Python Executable: /usr/bin/python3
Python Version: 3.6.9
Python Path:
['/home/project/agfzb-CloudAppDevelopment_Capstone/server',
'/usr/lib/python36.zip',
'/usr/lib/python3.6',
'/usr/lib/python3.6/lib-dynload',
'/home/theia/.local/lib/python3.6/site-packages',
'/usr/local/lib/python3.6/dist-packages',
'/usr/lib/python3/dist-packages']
Error message from Theia Labs:
File "/home/project/agfzb-CloudAppDevelopment_Capstone/server/djangoapp/views.py", line 42, in get_all_dealerships
dealer = get_all_dealerships_from_cf(dealer_url, id=id)
File "/home/project/agfzb-CloudAppDevelopment_Capstone/server/djangoapp/restapis.py", line 50, in get_all_dealerships_from_cf
dealers = json_result["entries"]
KeyError: 'entries'
from my views py file:
get_all_dealerships(request):
if request.method == "GET":
context = {}
dealer_url = "https://2d6871f8.us-south.apigw.appdomain.cloud/api/dealership/dealership"
apikey="xxxxxxxxxxxxxxx"
dealer = get_all_dealerships_from_cf(dealer_url, id=id)
context["dealer"] = dealer
review_url = "https://2d6871f8.us-south.apigw.appdomain.cloud/api/get_review/review"
reviews = get_allReviews_byID_from_cf(review_url, id=id)
print(reviews)
context["reviews"] = reviews
return render(request, 'djangoapp/dealer_details.html', context)
from my restapis file:
def get_all_dealerships_from_cf(url, **kwargs):
results = []
# Call get_request with a URL parameter
json_result = get_request(url)
if json_result:
# Get the row list in JSON as dealers
dealers = json_result["entries"]
# For each dealer object
for dealer in dealers:
# Get its content in `doc` object
dealer_doc = dealer["doc"]
# Create a CarDealer object with values in `doc` object
dealer_obj = CarDealer(address=dealer_doc["address"], city=dealer_doc["city"], id=dealer_doc["id"], lat=dealer_doc["lat"], long=dealer_doc["long"],
short_name=dealer_doc["short_name"],
st=dealer_doc["st"], zip=dealer_doc["zip"])
results.append(dealer_obj)
return results
Can't figure it out...

KeyError: 'apikey'

This is what the code is but getting keyerror "apikey"
class Firebase:
""" Firebase Interface """
def init(self, config):
self.api_key = config["apikey"]
self.auth_domain = config["authDomain"]
self.database_url = config["databaseURL"]
self.storage_bucket = config["storageBucket"]
self.credentials = None
self.requests = requests.Session()
if config.get("serviceAccount"):
scopes = [
'https://www.googleapis.com/auth/firebase.database',
'https://www.googleapis.com/auth/userinfo.email',
"https://www.googleapis.com/auth/cloud-platform"
]
service_account_type = type(config["serviceAccount"])
if service_account_type is str:
self.credentials = ServiceAccountCredentials.from_json_keyfile_name(config["serviceAccount"], scopes)
if service_account_type is dict:
self.credentials = ServiceAccountCredentials.from_json_keyfile_dict(config["serviceAccount"], scopes)
if is_appengine_sandbox():
# Fix error in standard GAE environment
# is releated to https://github.com/kennethreitz/requests/issues/3187
# ProtocolError('Connection aborted.', error(13, 'Permission denied'))
adapter = appengine.AppEngineAdapter(max_retries=3)
else:
adapter = requests.adapters.HTTPAdapter(max_retries=3)
for scheme in ('http://', 'https://'):
self.requests.mount(scheme, adapter)

Pool apply function hangs and never executes

I am trying to fetch Rally data by using its python library pyral. Sequentially the same code works, but its slow.
I thought of using python multiprocess package, however my pool.apply method gets stuck and never executes. I tried running it in Pycharm IDE as well as the windows cmd prompt.
import pandas as pd
from pyral import Rally
from multiprocessing import Pool, Manager
from pyral.entity import Project
def process_row(sheetHeaders: list, item: Project, L: list):
print('processing row : ' + item.Name) ## this print never gets called
row = ()
for header in sheetHeaders:
row.append(process_cell(header, item))
L.append(row)
def process_cell(attr, item: Project):
param = getattr(item, attr)
if param is None:
return None
try:
if attr == 'Owner':
return param.__getattr__('Name')
elif attr == 'Parent':
return param.__getattr__('ObjectID')
else:
return param
except KeyError as e:
print(e)
# Projects
# PortfolioItem
# User Story
# Hierarchical Req
# tasks
# defects
# -------------MAIN-----------------
def main():
# Rally connection
rally = Rally('rally1.rallydev.com', apikey='<my_key>')
file = 'rally_data.xlsx'
headers = {
'Project': ['Name', 'Description', 'CreationDate', 'ObjectID', 'Parent', 'Owner', 'State'],
}
sheetName = 'Project'
sheetHeaders = headers.get(sheetName)
p = Pool(1)
result = rally.get(sheetName, fetch=True, pagesize=10)
with Manager() as manager:
L = manager.list()
for item in result:
print('adding row for : ' + item.Name)
p.apply_async(func=process_row, args=(sheetHeaders, item, L)) ## gets stuck here
p.close()
p.join()
pd.DataFrame(L).to_excel(file, sheet_name=sheetName)
if __name__ == '__main__':
main()
Also tried without Manager list without any difference in the outcome
def main():
# Rally connection
rally = Rally('rally1.rallydev.com', apikey='<key>')
file = 'rally_data.xlsx'
headers = {
'Project': ['Name', 'Description', 'CreationDate', 'ObjectID', 'Parent', 'Owner', 'State'],
}
sheetName = 'Project'
sheetHeaders = headers.get(sheetName)
result = rally.get(sheetName, fetch=True, pagesize=10)
async_results = []
with Pool(50) as p:
for item in result:
print('adding row for : ' + item.Name)
async_results.append(p.apply_async(func=process_row, args=(sheetHeaders, item)))
res = [r.get() for r in async_results]
pd.DataFrame(res).to_excel(file, sheet_name=sheetName)
I dont know why, but replacing multiprocessing
with multiprocessing.dummy in the import statement worked.

Using Json Input Variables In Airflow EMR Operator Steps

I'm currently following the template given here: https://github.com/apache/airflow/blob/master/airflow/contrib/example_dags/example_emr_job_flow_manual_steps.py to create a DAG to call for a emr instance using spark submit. When setting up the spark_test_steps, I need to include variables passed in from a POST Json to fill the spark submit like below:
SPARK_TEST_STEPS = [
{
'Name': 'calculate_pi',
'ActionOnFailure': 'CONTINUE',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': [
'/usr/lib/spark/bin/run-example',
'SparkPi',
kwargs['dag_run'].conf['var_1']
kwargs['dag_run'].conf['var_2']
'10'
]
}
}
]
How can I pass in variables given by the POST Json while still following the format given in the git link to look like below?
from datetime import timedelta
import airflow
from airflow import DAG
from airflow.contrib.operators.emr_create_job_flow_operator import EmrCreateJobFlowOperator
from airflow.contrib.operators.emr_add_steps_operator import EmrAddStepsOperator
from airflow.contrib.sensors.emr_step_sensor import EmrStepSensor
from airflow.contrib.operators.emr_terminate_job_flow_operator import EmrTerminateJobFlowOperator
DEFAULT_ARGS = {
'owner': 'Airflow',
'depends_on_past': False,
'start_date': airflow.utils.dates.days_ago(2),
'email': ['airflow#example.com'],
'email_on_failure': False,
'email_on_retry': False
}
dag = DAG(
'emr_job_flow_manual_steps_dag',
default_args=DEFAULT_ARGS,
dagrun_timeout=timedelta(hours=2),
schedule_interval='0 3 * * *'
)
var_1 = ''
var_2 = ''
SPARK_TEST_STEPS = []
def define_param(**kwargs):
global var_1
global var_2
global SPARK_TEST_STEPS
var_1 = str(kwargs['dag_run'].conf['var_1'])
var_2 = str(kwargs['dag_run'].conf['var_2'])
SPARK_TEST_STEPS = [
{
'Name': 'calculate_pi',
'ActionOnFailure': 'CONTINUE',
'HadoopJarStep': {
'Jar': 'command-runner.jar',
'Args': [
'/usr/lib/spark/bin/run-example',
'SparkPi',
kwargs['dag_run'].conf['var_1']
kwargs['dag_run'].conf['var_2']
'10'
]
}
}
]
return SPARK_TEST_STEPS
DEFINE_PARAMETERS = PythonOperator(
task_id='DEFINE_PARAMETERS',
python_callable=define_param,
provide_context=True,
dag=dag)
cluster_creator = EmrCreateJobFlowOperator(
task_id='create_job_flow',
job_flow_overrides=JOB_FLOW_OVERRIDES,
aws_conn_id='aws_default',
emr_conn_id='emr_default',
dag=dag
)
step_adder = EmrAddStepsOperator(
task_id='add_steps',
job_flow_id="{{ task_instance.xcom_pull('create_job_flow', key='return_value') }}",
aws_conn_id='aws_default',
steps='{{ ti.xcom_pull(task_ids="DEFINE_PARAMETERS") }}',
dag=dag
)
step_checker = EmrStepSensor(
task_id='watch_step',
job_flow_id="{{ task_instance.xcom_pull('create_job_flow', key='return_value') }}",
step_id="{{ task_instance.xcom_pull('add_steps', key='return_value')[0] }}",
aws_conn_id='aws_default',
dag=dag
)
cluster_remover = EmrTerminateJobFlowOperator(
task_id='remove_cluster',
job_flow_id="{{ task_instance.xcom_pull('create_job_flow', key='return_value') }}",
aws_conn_id='aws_default',
dag=dag
)
cluster_creator.set_downstream(step_adder)
step_adder.set_downstream(step_checker)
step_checker.set_downstream(cluster_remover)
I cannot use Variable.get and Variable.set as this will not allow multiple dag calls for different variable types at the same time due to the constant changing of airflow global variables. I have tried calling SPARK_TEST_STEPS using xcom but the return type of xcom is string and EmrAddStepsOperator steps requires a list.
I solved a similar problem by creating a custom operator that parses the json prior to executing. The cause of the problem is that when you pass steps='{{ ti.xcom_pull(task_ids="DEFINE_PARAMETERS") }}',. you are literally passing a string with the value interpolated by the templating engine, it is not deserialized.
from airflow.contrib.hooks.emr_hook import EmrHook
from airflow.exceptions import AirflowException
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.contrib.operators.emr_add_steps_operator import EmrAddStepsOperator
import json
class DynamicEmrStepsOperator(EmrAddStepsOperator):
template_fields = ['job_flow_id', 'steps']
template_ext = ()
ui_color = '#f9c915'
#apply_defaults
def __init__(
self,
job_flow_id=None,
steps="[]",
*args, **kwargs):
super().__init__(
job_flow_id = job_flow_id,
steps = steps,
*args, **kwargs)
def execute(self, context):
self.steps = json.loads(self.steps)
return super().execute(context)

scrapy memory consumption is increasing only

I am using scrapy along with scrapyrt to create APIs. My application have around 20 spiders. We are using NFS server for load balancing on production. Unfortunately application using 40% and more space.
"stats": {
"downloader/request_bytes": 12033,
"downloader/request_count": 5,
"downloader/request_method_count/GET": 4,
"downloader/request_method_count/POST": 1,
"downloader/response_bytes": 20165,
"downloader/response_count": 5,
"downloader/response_status_count/200": 3,
"downloader/response_status_count/302": 1,
"downloader/response_status_count/404": 1,
"finish_reason": "finished",
"finish_time": "2019-05-23 06:05:04",
"item_scraped_count": 1,
"log_count/DEBUG": 35,
"log_count/INFO": 20,
"memusage/max": 3399057408,
"memusage/startup": 3399057408,
"request_depth_max": 2,
"response_received_count": 4,
"scheduler/dequeued": 4,
"scheduler/dequeued/memory": 4,
"scheduler/enqueued": 4,
"scheduler/enqueued/memory": 4,
"start_time": "2019-05-23 06:05:01"
}
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
14500 root 20 0 4999116 3.190g 7184 S 0.3 40.9 103:34.01 scrapyrt
I followed scrapy memory leak documentation so remove meta attribute from request, but still memory is increasing.
class GarudaRetrieveBookingSpider(Spider):
"""get the pax, flight and fare details"""
name = "garuda_retrieve_booking"
meta = dict()
formdata = dict()
start_url = ''
booking_code = ''
output_dict = {'schedule_detail': [], 'pax_details': [], 'reservation_name': '', 'fare_details': {}}
pax_count = 0
adult_count = 0
child_count = 0
infant_count = 0
ticket_list_adt_child = []
ticket_list_inf = []
# this variable is created to save rt command response data to pass in next call if there are no tickets
rt_response = ''
def start_requests(self):
"""
:return: Request object
"""
post_data = self.data
garuda_session_id = post_data['parameter']['jSessionId']
post_data["command"] = "IG"
file_path = os.path.join(GARUDA_SESSION_FILES_PATH, TODAY_DATE, garuda_session_id + "_session.txt")
session_data = get_cookies(self, file_path)
self.start_url = GARUDA_KEEP_ALIVE_URL.format(session_id=session_data["jSessionId"], site=SITE, lang=LANGUAGE)
self.meta = {"session_data": session_data, "post_data": post_data}
return [Request(self.start_url, self.parse, errback=self.errback_httpbin)]
def parse(self, response):
"""
:param response:
:return: FormRequest
description: submit IG command
"""
self.log("\n\nparse response: {}\n\n".format(response.text))
if response.status != 200:
error_message = 'parse method failed.'
return {"status": False, "error_message": error_message}
session_data = self.meta["session_data"]
command = self.meta["post_data"]["command"]
# override the command with current command
session_data["tasks"][0]["command"]["command"] = command
self.formdata = {
"data": json.dumps(session_data)
}
yield scrapy.FormRequest(CRYTO_COMMAND_URL, formdata=self.formdata,
callback=self.ig_command_response, errback=self.errback_httpbin)