How to create an invoice and invoice lines via code - Odoo13 - odoo

I am trying to create an invoice and invoice lines via python code.
Here is the code.
def createInvoice(self, date_ref):
// Skipped some code here
invoice_values = contract._prepare_invoice(date_ref)
for line in contract_lines:
invoice_values.setdefault('invoice_line_ids', [])
invoice_line_values = line._prepare_invoice_line(
invoice_id=False
)
if invoice_line_values:
invoice_values['invoice_line_ids'].append(
(0, 0, invoice_line_values)
)
invoices_values.append(invoice_values)
​
Values for
invoice_values = {'type': 'in_invoice', 'journal_id': 2, 'company_id': 1, 'line_ids': [(6, 0, [])],
'partner_id': 42, 'commercial_partner_id': 42, 'fiscal_position_id': False,
'invoice_payment_term_id': False, 'invoice_line_ids': [(6, 0, [])],
'invoice_partner_bank_id': False, 'invoice_cash_rounding_id': False,
'bank_partner_id': 42, 'currency_id': 130, 'invoice_date': datetime.date(2020, 11, 11),
'invoice_origin': 'Vendor COntract #1', 'user_id': 2, 'old_contract_id': 6}
invoice_line_values = {'move_id': False, 'journal_id': False, 'company_id': False,
'account_id': False, 'name': '[E-COM07] Large Cabinet', 'quantity': 1.0,
'price_unit': 1444.01, 'discount': 0.0, 'partner_id': False,
'product_uom_id': 1, 'product_id': 17, 'payment_id': False,
'tax_ids': [(6, 0, [])], 'analytic_line_ids': [(6, 0, [])],
'display_type': False, 'contract_line_id': 7, 'asset_id': False,
'analytic_account_id': False}
In create function of account move
vals = {'date': datetime.date(2020, 2, 11), 'type': 'in_invoice', 'journal_id': 2,
'company_id': 1, 'currency_id': 130, 'line_ids': [(6, 0, [])], 'partner_id': 42,
'commercial_partner_id': 42, 'fiscal_position_id': False, 'user_id': 2,
'invoice_date': datetime.date(2020, 12, 11), 'invoice_origin': 'Vendor COntract #1',
'invoice_payment_term_id': False,
'invoice_line_ids': [(6, 0, []), (0, 0, {'journal_id': False, 'company_id': False,
'account_id': 109, 'name': '[E-COM07] Large Cabinet', 'quantity': 1.0, 'price_unit': 1444.01,
'discount': 0.0, 'partner_id': False, 'product_uom_id': 1, 'product_id': 17,
'payment_id': False, 'tax_ids': [(6, 0, [19])], 'analytic_line_ids': [(6, 0, [])],
'analytic_account_id': False, 'display_type': False, 'exclude_from_invoice_tab': False,
'contract_line_id': 7, 'asset_id': False}), (0, 0, {'journal_id': False,
'company_id': False, 'account_id': 109, 'name': '[E-COM09] Large Desk', 'quantity': 1.0,
'price_unit': 8118.04, 'discount': 0.0, 'partner_id': False, 'product_uom_id': 1,
'product_id': 19, 'payment_id': False, 'tax_ids': [(6, 0, [19])],
'analytic_line_ids': [(6, 0, [])], 'analytic_account_id': False, 'display_type': False,
'exclude_from_invoice_tab': False, 'contract_line_id': 8, 'asset_id': False})],
'invoice_partner_bank_id': False, 'invoice_cash_rounding_id': False, 'bank_partner_id': 42,
'old_contract_id': 6}
It creates the account_move(Invoice) but not the account_move_line(Invoice lines).
What am i missing here?

Finally got the solution,
'line_ids': [(6, 0, [])],
The above line caused the problem. I removed it from the invoice_values then it worked.

Related

Bar of proportion of two variables

I am having a pandas dataframe as shown below
import numpy as np
data = {
'id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50],
'baseline': [1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1],
'endline': [1, 0, np.nan, 1, 0, 0, 1, np.nan, 1, 0, 0, 1, 0, 0, 1, 0, np.nan, np.nan, 1, 0, 1, np.nan, 0, 1, 0, 1, 0, np.nan, 1, 0, np.nan, 0, 0, 0, np.nan, 1, np.nan, 1, np.nan, 0, np.nan, 1, 1, 0, 1, 1, 1, 0, 1, 1],
'gender': ['male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'male', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female', 'female']
}
df = pd.DataFrame(data)
df.head(n = 5)
The challenge is the endline column may have some missing values. My goal is to have 2 bars for each variable side by side as shown below.
Thanks in advance!
Seaborn prefers its data in "long form". Pandas' melt can convert the given dataframe to combine the 'baseline' and 'endline' columns.
By default, sns.barplot shows the mean when there are multiple y-values belonging to the same x-value. You can use a different estimator, e.g. summing the values and dividing by the number of values to get a percentage.
Here is some code to get you started:
import matplotlib.pyplot as plt
from matplotlib.ticker import PercentFormatter
import seaborn as sns
import pandas as pd
import numpy as np
data = {
'id': range(1, 51),
'baseline': [1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1],
'endline': [1, 0, np.nan, 1, 0, 0, 1, np.nan, 1, 0, 0, 1, 0, 0, 1, 0, np.nan, np.nan, 1, 0, 1, np.nan, 0, 1, 0, 1, 0, np.nan, 1, 0, np.nan, 0, 0, 0, np.nan, 1, np.nan, 1, np.nan, 0, np.nan, 1, 1, 0, 1, 1, 1, 0, 1, 1]
}
df = pd.DataFrame(data)
sns.set_style('white')
ax = sns.barplot(data=df.melt(value_vars=['baseline', 'endline']),
x='variable', y='value',
estimator=lambda x: np.sum(x) / np.size(x) * 100, ci=None,
color='cornflowerblue')
ax.bar_label(ax.containers[0], fmt='%.1f %%', fontsize=20)
sns.despine(ax=ax, left=True)
ax.grid(True, axis='y')
ax.yaxis.set_major_formatter(PercentFormatter(100))
ax.set_xlabel('')
ax.set_ylabel('')
plt.tight_layout()
plt.show()

How to randomly sample pairs of elements from a masked numpy array until the exhaustion of elements without any repetition?

I have an array of integers/indices (i.e. arr_F_idx = np.arange(0,200)) and I would like to draw 100 pairs without repetition. I am using a masked array (I transform arr_F_idx after the first draw, as shown in the code below), but it seems that numpy.random.choice still draws the masked elements.
arr_F_idx = np.arange(0,200)
draw = np.random.choice(arr_F_idx,2,replace=False)
arr2_Drawn_pairs[0,0] = draw[0]
arr2_Drawn_pairs[0,1] = draw[1]
arr_F_idx_dum1 = np.array(arr_F_idx == draw[0])
arr_F_idx = np.ma.array(arr_F_idx, mask = arr_F_idx_dum1)
arr_F_idx_dum2 = np.array(arr_F_idx == draw[1])
arr_F_idx = np.ma.array(arr_F_idx, mask = arr_F_idx_dum2)
for i in range(1,100):
draw = np.random.choice(arr_F_idx,2,replace=False)
arr2_Drawn_pairs[i,0] = draw[0]
arr2_Drawn_pairs[i,1] = draw[1]
arr_F_idx_dum1 = np.array(arr_F_idx == draw[0])
arr_F_idx = np.ma.array(arr_F_idx, mask = arr_F_idx_dum1)
arr_F_idx_dum2 = np.array(arr_F_idx == draw[1])
arr_F_idx = np.ma.array(arr_F_idx, mask = arr_F_idx_dum2)
The (sample) output that I get is
arr_F_idx
masked_array(data=[--, --, --, --, 4, 5, --, --, --, --, --, 11, 12, 13,
--, --, 16, 17, --, 19, --, 21, 22, --, 24, --, --, --,
28, --, --, --, 32, 33, 34, --, --, 37, 38, 39, --, 41,
--, --, --, --, --, --, 48, --, --, --, --, --, --, --,
--, 57, 58, --, 60, --, --, 63, 64, --, --, --, --, --,
--, --, --, 73, --, --, 76, --, 78, --, --, --, --, --,
--, --, --, --, 88, 89, --, --, 92, --, --, --, --, --,
98, 99, --, 101, 102, --, --, --, 106, --, --, --, --,
111, --, --, --, --, 116, --, --, --, --, 121, --, 123,
124, 125, --, 127, --, --, --, 131, --, --, --, 135,
--, --, --, --, --, 141, --, --, --, --, --, --, --,
--, --, 151, --, --, --, 155, --, --, --, 159, --, 161,
--, --, --, 165, --, 167, --, 169, --, 171, --, --, --,
--, 176, --, 178, 179, --, --, --, 183, --, 185, --,
--, --, 189, 190, --, --, 193, 194, --, 196, --, 198,
199],
mask=[ True, True, True, True, False, False, True, True,
True, True, True, False, False, False, True, True,
False, False, True, False, True, False, False, True,
False, True, True, True, False, True, True, True,
False, False, False, True, True, False, False, False,
True, False, True, True, True, True, True, True,
False, True, True, True, True, True, True, True,
True, False, False, True, False, True, True, False,
False, True, True, True, True, True, True, True,
True, False, True, True, False, True, False, True,
True, True, True, True, True, True, True, True,
False, False, True, True, False, True, True, True,
True, True, False, False, True, False, False, True,
True, True, False, True, True, True, True, False,
True, True, True, True, False, True, True, True,
True, False, True, False, False, False, True, False,
True, True, True, False, True, True, True, False,
True, True, True, True, True, False, True, True,
True, True, True, True, True, True, True, False,
True, True, True, False, True, True, True, False,
True, False, True, True, True, False, True, False,
True, False, True, False, True, True, True, True,
False, True, False, False, True, True, True, False,
True, False, True, True, True, False, False, True,
True, False, False, True, False, True, False, False],
fill_value=999999)
For smaller ranges it happens as well; of course for very small ranges this is not a problem, but as I have mentioned, I want to exhaust the original array to the point that all its elements will be masked.
The problem seems to be that the np.random.choice somehow still draws the masked elements, even though it is not supposed to (otherwise I do not see the point of the object called masked array). I may be doing something wrong. I will appreciate help on this issue, also if there is a simpler way to make the draws of pairs without repetition across and within pairs.
Edit: In fact, the numpy random.choice draws masked elements, as can be seen in the output (e.g. number 177 is drawn twice and 191 three times):
arr2_Drawn_pairs
Out[53]:
array([[ 20., 49.],
[ 35., 114.],
[ 44., 42.],
[ 52., 140.],
[191., 59.], 191 - the first time
[147., 144.],
[ 74., 143.],
[ 23., 43.],
[130., 1.],
[146., 166.],
[ 62., 80.],
[ 26., 138.],
[152., 71.],
[ 50., 87.],
[ 69., 9.],
[ 20., 65.],
[ 3., 162.],
[ 30., 104.],
[168., 145.],
[154., 54.],
[129., 2.],
[ 79., 170.],
[ 14., 188.],
[107., 30.],
[119., 188.],
[139., 94.],
[132., 158.],
[ 0., 69.],
[ 47., 27.],
[192., 72.],
[181., 160.],
[ 95., 162.],
[ 40., 25.],
[107., 8.],
[128., 10.],
[ 7., 83.],
[ 91., 173.],
[174., 10.],
[134., 82.],
[ 67., 52.],
[195., 172.],
[197., 96.],
[ 15., 188.],
[184., 164.],
[ 18., 180.],
[ 45., 27.],
[ 86., 84.],
[ 97., 128.],
[149., 6.],
[109., 85.],
[182., 62.],
[ 53., 68.],
[157., 81.],
[188., 25.],
[107., 45.],
[117., 86.],
[195., 47.],
[105., 103.],
[ 51., 162.],
[187., 162.],
[ 70., 97.],
[ 29., 156.],
[175., 177.],
[ 0., 10.],
[ 87., 46.],
[ 1., 119.],
[ 93., 90.],
[174., 53.],
[ 77., 84.],
[ 84., 66.],
[ 91., 186.],
[ 83., 59.],
[137., 140.],
[136., 186.],
[100., 195.],
[173., 81.],
[120., 115.],
[ 36., 46.],
[112., 148.],
[118., 103.],
[ 8., 128.],
[ 56., 65.],
[158., 145.],
[180., 122.],
[142., 126.],
[133., 45.],
[ 59., 173.],
[110., 119.],
[177., 31.], 177 - the first time!
[ 82., 158.],
[ 53., 113.],
[ 85., 150.],
[126., 94.],
[ 61., 152.],
[ 93., 40.],
[ 1., 55.],
[ 96., 162.],
[153., 108.],
[163., 9.],
[ 75., 50.],
[101., 47.],
[178., 148.],
[188., 183.],
[ 69., 177.], 177 - the second time!
[141., 16.],
[ 31., 28.],
[106., 147.],
[ 66., 176.],
[156., 96.],
[ 9., 21.],
[139., 57.],
[106., 11.],
[ 25., 2.],
[152., 69.],
[ 34., 169.],
[148., 191.], 191 - the second time!
[105., 32.],
[187., 156.],
[105., 191.], 191 - the third time!
[ 53., 128.],
[ 56., 30.],
[176., 7.],
[168., 150.],
[ 48., 101.],
[105., 167.]])
Edit 2: A brute-force method for obtaining my desired result is perhaps creating a new array to sample from after every draw,
arr_F_idx_iter = np.append(arr_F_idx[0:draw[0]],arr_F_idx[draw[0]+1:199])
but I think the question of whether this can be done efficiently with masked arrays is still legitimate, as it would be quicker, maybe also points out a flaw in how masked arrays work.
A simple 1d masked array:
In [28]: m = np.ma.masked_array(np.arange(10), mask=np.random.randint(0,2,10))
In [29]: m
Out[29]:
masked_array(data=[--, --, 2, 3, --, --, 6, 7, --, --],
mask=[ True, True, False, False, True, True, False, False,
True, True],
fill_value=999999)
choice, without special ma knowledge, draws from the data attribute:
In [31]: np.random.choice(m,3, replace=False)
Out[31]: array([4, 3, 8])
In [32]: np.random.choice(m.data,3, replace=False)
Out[32]: array([1, 8, 5])
If you want it to draw from the unmasked elements, you need to give it such an array, compressed:
In [33]: np.random.choice(m.compressed(),3, replace=False)
Out[33]: array([2, 6, 3])
In general np functions don't work correctly on masked arrays. That's why there's a large set of np.ma functions (and ma methods). Often those functions use the compressed to get the unmasked values. Or they replace masked values with some "innocent" fill.
In [35]: m.data
Out[35]: array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
In [36]: m.compressed()
Out[36]: array([2, 3, 6, 7])
In [37]: m.filled()
Out[37]:
array([999999, 999999, 2, 3, 999999, 999999, 6, 7,
999999, 999999])
In [38]: m.filled(0)
Out[38]: array([0, 0, 2, 3, 0, 0, 6, 7, 0, 0])

How to make selenium page loading more efficient if accessing all data on the page requires 10k+ clicks of a lazyload button?

I am scraping one particular page with the a headless chromedriver
The page is really huge, to load it entirely I need 10k+ clicks on a lazy load button
The more I click, the slower things get
Is there a way to make the process faster?
Here is the code:
def driver_config():
chrome_options = Options()
prefs = {"profile.managed_default_content_settings.images": 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.page_load_strategy = 'eager'
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(options=chrome_options)
return(driver)
def scroll_the_category_until_the_end(driver, category_url):
driver.get(category_url)
pbar = tqdm()
pbar.write('initializing spin')
while True:
try:
show_more_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="root"]/div/div[2]/div[2]/div[2]/button')))
driver.execute_script("arguments[0].click();", show_more_button)
pbar.update()
except TimeoutException:
pbar.write('docking')
pbar.close()
break
driver = driver_config()
scroll_the_category_until_the_end(driver, 'https://supl.biz/russian-federation/stroitelnyie-i-otdelochnyie-materialyi-supplierscategory9403/')
UPDATE:
I also tried to implement another strategy but it didn't work:
deleting all company information on every iteration
clearing driver cash
My hypothesis was that if I do this, DOM will always be clean and fast
driver = driver_config()
driver.get('https://supl.biz/russian-federation/stroitelnyie-i-otdelochnyie-materialyi-supplierscategory9403/')
pbar = tqdm()
pbar.clear()
while True:
try:
for el in driver.find_elements_by_class_name('a_zvOKG8vZ'):
driver.execute_script("""var element = arguments[0];element.parentNode.removeChild(element);""", el)
button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH,"//*[contains(text(), 'Показать больше поставщиков')]")))
driver.execute_script("arguments[0].click();", button)
pbar.update()
driver.execute_script("window.localStorage.clear();")
except Exception as e:
pbar.close()
print(e)
break
First the website invokes javascript to grab new data, the HTTP request is invoked by clicking the more results button, it calls upon an API and the response back is the data needed to load the page with more results. You can view this request by inspecting the page --> Network tools --> XHR and then clicking the button. It sends an HTTP GET request to an API which has data on each product.
The most efficient way to grab data from a website that invokes javascript is by re-engineering this HTTP request the javascript is making.
In this case it's relatively easy, I copied the request in a cURL command within XHR of inspecting the page and converted it using curl.trillworks.com to python.
This is the screen you get to with XHR, before clicking the more results page.
Clicking the more results page you get this, notice how a request has populated the screen ?
Here I'm copying the cURL request to grab the necessary headers etc...
Here I'm copying the cURL request to grab the necessary headers etc... you can then input this into curl.trillworks.com and it converts the request into params, cookies and headers and gives you boilerplate for the requests package.
Had a play around with the request using the requests package. Inputting various parts of the headers, you are provided cookies, but they're actually not necessary when you make the request.
The simplest request to make is one without headers, parameters or cookies but most API endpoints don't accept this. In this case, having played around with the requests package, you need a user-agent and the parameters that specify what data you get back from the API. Infact you don't even need a valid user-agent.
Now you could invoke a while loop to keep making HTTP requests in sizes of 8. Unfortunately altering the size of the request in the parameters doesn't get you all the data!
Coding Example
import requests
import time
i = 8
j = 1
headers = {
'user-agent': 'M'
}
while True:
if response.status_code == 200:
params = (
('category', '9403'),
('city', 'russian-federation'),
('page', f'{j}'),
('size', f'{i}'),
)
response = requests.get('https://supl.biz/api/monolith/suppliers-catalog/search/', headers=headers, params=params)
print(response.json()['hits'][0])
i += 8
j += 1
time.sleep(4)
else:
break
Output
Sample output
{'id': 1373827,
'type': None,
'highlighted': None,
'count_days_on_tariff': 183,
'tariff_info': {'title_for_show': 'Поставщик Премиум',
'finish_date': '2021-02-13',
'url': '/supplier-premium-membership/',
'start_date': '2020-08-13'},
'origin_ru': {'id': 999, 'title': 'Санкт-Петербург'},
'title': 'ООО "СТАНДАРТ 10"',
'address': 'Пискаревский проспект, 150, корпус 2.',
'inn': '7802647317',
'delivery_types': ['self', 'transportcompany', 'suppliercars', 'railway'],
'summary': 'Сэндвич-панели: новые, 2й сорт, б/у. Холодильные камеры: новые, б/у. Двери для холодильных камер: новые, б/у. Строительство холодильных складов, ангаров и др. коммерческих объектов из сэндвич-панелей. Холодильное оборудование: новое, б/у.',
'phone': '79219602762',
'hash_id': 'lMJPgpEz7b',
'payment_types': ['cache', 'noncache'],
'logo_url': 'https://suplbiz-a.akamaihd.net/media/cache/37/9e/379e9fafdeaab4fc5a068bc90845b56b.jpg',
'proposals_count': 4218,
'score': 42423,
'reviews': 0,
'rating': '0.0',
'performed_orders_count': 1,
'has_replain_chat': False,
'verification_status': 2,
'proposals': [{'id': 20721916,
'title': 'Сэндвич панели PIR 100',
'description': 'Сэндвич панели. Наполнение Пенополиизлцианурат ПИР PIR. Толщина 100мм. Длина 3,2 метра. Rall9003/Rall9003. Вналичии 600м2. Количество: 1500',
'categories': [135],
'price': 1250.0,
'old_price': None,
'slug': 'sendvich-paneli-pir-100',
'currency': 'RUB',
'price_details': 'Цена за шт.',
'image': {'preview_220x136': 'https://suplbiz-a.akamaihd.net/media/cache/72/4d/724d0ba4d4a2b7d459f3ca4416e58d7d.jpg',
'image_dominant_color': '#ffffff',
'preview_140': 'https://suplbiz-a.akamaihd.net/media/cache/67/45/6745bb6f616b82f7cd312e27814b6b89.jpg',
'hash': 'd41d8cd98f00b204e9800998ecf8427e'},
'additional_images': [],
'availability': 1,
'views': 12,
'seo_friendly': False,
'user': {'id': 1373827,
'name': 'ООО "СТАНДАРТ 10"',
'phone': '+79219602762',
'address': 'Пискаревский проспект, 150, корпус 2.',
'origin_id': 999,
'country_id': 1,
'origin_title': 'Санкт-Петербург',
'verified': False,
'score': 333,
'rating': 0.0,
'reviews': 0,
'tariff': {'title_for_show': 'Поставщик Премиум',
'count_days_on_tariff': 183,
'finish_date': '2021-02-13',
'url': '/supplier-premium-membership/',
'start_date': '2020-08-13'},
'performed_orders_count': 1,
'views': 12,
'location': {'lon': 30.31413, 'lat': 59.93863}}},
{'id': 20722131,
'title': 'Сэндвич панели ппу100 б/у, 2,37 м',
'description': 'Сэндвич панели. Наполнение Пенополиуретан ППУ ПУР PUR. Толщина 100 мм. длинна 2,37 метра. rall9003/rall9003. БУ. В наличии 250 м2.',
'categories': [135],
'price': 800.0,
'old_price': None,
'slug': 'sendvich-paneli-ppu100-b-u-2-37-m',
'currency': 'RUB',
'price_details': 'Цена за шт.',
'image': {'preview_220x136': 'https://suplbiz-a.akamaihd.net/media/cache/d1/49/d1498144bc7b324e288606b0d7d98120.jpg',
'image_dominant_color': '#ffffff',
'preview_140': 'https://suplbiz-a.akamaihd.net/media/cache/10/4b/104b54cb9b7ddbc6b2f0c1c5a01cdc2d.jpg',
'hash': 'd41d8cd98f00b204e9800998ecf8427e'},
'additional_images': [],
'availability': 1,
'views': 4,
'seo_friendly': False,
'user': {'id': 1373827,
'name': 'ООО "СТАНДАРТ 10"',
'phone': '+79219602762',
'address': 'Пискаревский проспект, 150, корпус 2.',
'origin_id': 999,
'country_id': 1,
'origin_title': 'Санкт-Петербург',
'verified': False,
'score': 333,
'rating': 0.0,
'reviews': 0,
'tariff': {'title_for_show': 'Поставщик Премиум',
'count_days_on_tariff': 183,
'finish_date': '2021-02-13',
'url': '/supplier-premium-membership/',
'start_date': '2020-08-13'},
'performed_orders_count': 1,
'views': 4,
'location': {'lon': 30.31413, 'lat': 59.93863}}},
{'id': 20722293,
'title': 'Холодильная камера polair 2.56х2.56х2.1',
'description': 'Холодильная камера. Размер 2,56 Х 2,56 Х 2,1. Камера из сэндвич панелей ППУ80. Камера с дверью. -5/+5 или -18. В наличии. Подберем моноблок или сплит систему. …',
'categories': [478],
'price': 45000.0,
'old_price': None,
'slug': 'holodilnaya-kamera-polair-2-56h2-56h2-1',
'currency': 'RUB',
'price_details': 'Цена за шт.',
'image': {'preview_220x136': 'https://suplbiz-a.akamaihd.net/media/cache/c1/9f/c19f38cd6893a3b94cbdcbdb8493c455.jpg',
'image_dominant_color': '#ffffff',
'preview_140': 'https://suplbiz-a.akamaihd.net/media/cache/4d/b0/4db06a2508cccf5b2e7fe822c1b892a2.jpg',
'hash': 'd41d8cd98f00b204e9800998ecf8427e'},
'additional_images': [],
'availability': 1,
'views': 5,
'seo_friendly': False,
'user': {'id': 1373827,
'name': 'ООО "СТАНДАРТ 10"',
'phone': '+79219602762',
'address': 'Пискаревский проспект, 150, корпус 2.',
'origin_id': 999,
'country_id': 1,
'origin_title': 'Санкт-Петербург',
'verified': False,
'score': 333,
'rating': 0.0,
'reviews': 0,
'tariff': {'title_for_show': 'Поставщик Премиум',
'count_days_on_tariff': 183,
'finish_date': '2021-02-13',
'url': '/supplier-premium-membership/',
'start_date': '2020-08-13'},
'performed_orders_count': 1,
'views': 5,
'location': {'lon': 30.31413, 'lat': 59.93863}}},
{'id': 20722112,
'title': 'Сэндвич панели ппу 80 б/у, 2,4 м',
'description': 'Сэндвич панели. Наполнение ППУ. Толщина 80 мм. длинна 2,4 метра. БУ. В наличии 350 м2.',
'categories': [135],
'price': 799.0,
'old_price': None,
'slug': 'sendvich-paneli-ppu-80-b-u-2-4-m',
'currency': 'RUB',
'price_details': 'Цена за шт.',
'image': {'preview_220x136': 'https://suplbiz-a.akamaihd.net/media/cache/ba/06/ba069a73eda4641030ad69633d79675d.jpg',
'image_dominant_color': '#ffffff',
'preview_140': 'https://suplbiz-a.akamaihd.net/media/cache/4f/e9/4fe9f3f358f775fa828c532a6c08e7f2.jpg',
'hash': 'd41d8cd98f00b204e9800998ecf8427e'},
'additional_images': [],
'availability': 1,
'views': 8,
'seo_friendly': False,
'user': {'id': 1373827,
'name': 'ООО "СТАНДАРТ 10"',
'phone': '+79219602762',
'address': 'Пискаревский проспект, 150, корпус 2.',
'origin_id': 999,
'country_id': 1,
'origin_title': 'Санкт-Петербург',
'verified': False,
'score': 333,
'rating': 0.0,
'reviews': 0,
'tariff': {'title_for_show': 'Поставщик Премиум',
'count_days_on_tariff': 183,
'finish_date': '2021-02-13',
'url': '/supplier-premium-membership/',
'start_date': '2020-08-13'},
'performed_orders_count': 1,
'views': 8,
'location': {'lon': 30.31413, 'lat': 59.93863}}},
{'id': 20722117,
'title': 'Сэндвич панели ппу 60 мм, 2,99 м',
'description': 'Сэндвич панели. Наполнение Пенополиуретан ППУ ПУР PUR . Новые. В наличии 600 м2. Толщина 60 мм. длинна 2,99 метров. rall9003/rall9003',
'categories': [135],
'price': 1100.0,
'old_price': None,
'slug': 'sendvich-paneli-ppu-60-mm-2-99-m',
'currency': 'RUB',
'price_details': 'Цена за шт.',
'image': {'preview_220x136': 'https://suplbiz-a.akamaihd.net/media/cache/e2/fb/e2fb6505a5af74a5a994783a5e51600c.jpg',
'image_dominant_color': '#ffffff',
'preview_140': 'https://suplbiz-a.akamaihd.net/media/cache/9c/f5/9cf5905a26e6b2ea1fc16d50c19ef488.jpg',
'hash': 'd41d8cd98f00b204e9800998ecf8427e'},
'additional_images': [],
'availability': 1,
'views': 10,
'seo_friendly': False,
'user': {'id': 1373827,
'name': 'ООО "СТАНДАРТ 10"',
'phone': '+79219602762',
'address': 'Пискаревский проспект, 150, корпус 2.',
'origin_id': 999,
'country_id': 1,
'origin_title': 'Санкт-Петербург',
'verified': False,
'score': 333,
'rating': 0.0,
'reviews': 0,
'tariff': {'title_for_show': 'Поставщик Премиум',
'count_days_on_tariff': 183,
'finish_date': '2021-02-13',
'url': '/supplier-premium-membership/',
'start_date': '2020-08-13'},
'performed_orders_count': 1,
'views': 10,
'location': {'lon': 30.31413, 'lat': 59.93863}}}]}
​
Explanation
Here we're making sure that the response status is 200 before making another request. Using f-strings we change the page by 1 and the size of the results of the JSON object by 8 each iteration of the while loop. I've imposed a time restriction per request, because if push too many HTTP request at once you'll end up getting IP banned. Be gentle on the server!
The response.json() method converts the JSON object to python dictionary, you haven't specified what data, but I think if you can handle a python dictionary you can grab the data you require.
Comments
Here is where the parameters comes from. You can see the pages and size data here.

Jupyter .save_to_html function does not store config

I'm trying to use the .save_to_html() function for a kepler.gl jupyter notebook map.
It all works great inside jupyter, and I can re-load the same map with a defined config.
Where it goes wrong is when I use the save_to_html() function. The map will save to an html, but the configuration reverts to the basic configuration, before I customized it.
Please help! I love kepler, when I solve this little thing, it will be our absolute go-to tool
Thanks
Have tried to change the filters, colours, and point sizes. None of this works.
map_1 = KeplerGl(height=500, data={'data': df},config=config)
map_1
config = map_1.config
config
map_1.save_to_html(data={'data_1': df},
file_name='privateers.html',config=config)
Config
{'version': 'v1',
'config': {'visState': {'filters': [{'dataId': 'data',
'id': 'x8t9c53mf',
'name': 'time_update',
'type': 'timeRange',
'value': [1565687902187.5417, 1565775465282],
'enlarged': True,
'plotType': 'histogram',
'yAxis': None},
{'dataId': 'data',
'id': 'biysqlu36',
'name': 'user_id',
'type': 'multiSelect',
'value': ['HNc0SI3WsQfhOFRF2THnUEfmqJC3'],
'enlarged': False,
'plotType': 'histogram',
'yAxis': None}],
'layers': [{'id': 'ud6168',
'type': 'point',
'config': {'dataId': 'data',
'label': 'Point',
'color': [18, 147, 154],
'columns': {'lat': 'lat', 'lng': 'lng', 'altitude': None},
'isVisible': True,
'visConfig': {'radius': 5,
'fixedRadius': False,
'opacity': 0.8,
'outline': False,
'thickness': 2,
'strokeColor': None,
'colorRange': {'name': 'Uber Viz Qualitative 1.2',
'type': 'qualitative',
'category': 'Uber',
'colors': ['#12939A',
'#DDB27C',
'#88572C',
'#FF991F',
'#F15C17',
'#223F9A'],
'reversed': False},
'strokeColorRange': {'name': 'Global Warming',
'type': 'sequential',
'category': 'Uber',
'colors': ['#5A1846',
'#900C3F',
'#C70039',
'#E3611C',
'#F1920E',
'#FFC300']},
'radiusRange': [0, 50],
'filled': True},
'textLabel': [{'field': None,
'color': [255, 255, 255],
'size': 18,
'offset': [0, 0],
'anchor': 'start',
'alignment': 'center'}]},
'visualChannels': {'colorField': {'name': 'ride_id', 'type': 'string'},
'colorScale': 'ordinal',
'strokeColorField': None,
'strokeColorScale': 'quantile',
'sizeField': None,
'sizeScale': 'linear'}},
{'id': 'an8tbef',
'type': 'point',
'config': {'dataId': 'data',
'label': 'previous',
'color': [221, 178, 124],
'columns': {'lat': 'previous_lat',
'lng': 'previous_lng',
'altitude': None},
'isVisible': False,
'visConfig': {'radius': 10,
'fixedRadius': False,
'opacity': 0.8,
'outline': False,
'thickness': 2,
'strokeColor': None,
'colorRange': {'name': 'Global Warming',
'type': 'sequential',
'category': 'Uber',
'colors': ['#5A1846',
'#900C3F',
'#C70039',
'#E3611C',
'#F1920E',
'#FFC300']},
'strokeColorRange': {'name': 'Global Warming',
'type': 'sequential',
'category': 'Uber',
'colors': ['#5A1846',
'#900C3F',
'#C70039',
'#E3611C',
'#F1920E',
'#FFC300']},
'radiusRange': [0, 50],
'filled': True},
'textLabel': [{'field': None,
'color': [255, 255, 255],
'size': 18,
'offset': [0, 0],
'anchor': 'start',
'alignment': 'center'}]},
'visualChannels': {'colorField': None,
'colorScale': 'quantile',
'strokeColorField': None,
'strokeColorScale': 'quantile',
'sizeField': None,
'sizeScale': 'linear'}},
{'id': 'ilpixu9',
'type': 'arc',
'config': {'dataId': 'data',
'label': ' -> previous arc',
'color': [146, 38, 198],
'columns': {'lat0': 'lat',
'lng0': 'lng',
'lat1': 'previous_lat',
'lng1': 'previous_lng'},
'isVisible': True,
'visConfig': {'opacity': 0.8,
'thickness': 2,
'colorRange': {'name': 'Global Warming',
'type': 'sequential',
'category': 'Uber',
'colors': ['#5A1846',
'#900C3F',
'#C70039',
'#E3611C',
'#F1920E',
'#FFC300']},
'sizeRange': [0, 10],
'targetColor': None},
'textLabel': [{'field': None,
'color': [255, 255, 255],
'size': 18,
'offset': [0, 0],
'anchor': 'start',
'alignment': 'center'}]},
'visualChannels': {'colorField': None,
'colorScale': 'quantile',
'sizeField': None,
'sizeScale': 'linear'}},
{'id': 'inv52pp',
'type': 'line',
'config': {'dataId': 'data',
'label': ' -> previous line',
'color': [136, 87, 44],
'columns': {'lat0': 'lat',
'lng0': 'lng',
'lat1': 'previous_lat',
'lng1': 'previous_lng'},
'isVisible': False,
'visConfig': {'opacity': 0.8,
'thickness': 2,
'colorRange': {'name': 'Global Warming',
'type': 'sequential',
'category': 'Uber',
'colors': ['#5A1846',
'#900C3F',
'#C70039',
'#E3611C',
'#F1920E',
'#FFC300']},
'sizeRange': [0, 10],
'targetColor': None},
'textLabel': [{'field': None,
'color': [255, 255, 255],
'size': 18,
'offset': [0, 0],
'anchor': 'start',
'alignment': 'center'}]},
'visualChannels': {'colorField': None,
'colorScale': 'quantile',
'sizeField': None,
'sizeScale': 'linear'}}],
'interactionConfig': {'tooltip': {'fieldsToShow': {'data': ['time_ride_start',
'user_id',
'ride_id']},
'enabled': True},
'brush': {'size': 0.5, 'enabled': False}},
'layerBlending': 'normal',
'splitMaps': []},
'mapState': {'bearing': 0,
'dragRotate': False,
'latitude': 49.52565611453996,
'longitude': 6.2730441822977845,
'pitch': 0,
'zoom': 9.244725880765998,
'isSplit': False},
'mapStyle': {'styleType': 'dark',
'topLayerGroups': {},
'visibleLayerGroups': {'label': True,
'road': True,
'border': False,
'building': True,
'water': True,
'land': True,
'3d building': False},
'threeDBuildingColor': [9.665468314072013,
17.18305478057247,
31.1442867897876],
'mapStyles': {}}}}
Expected:
Fully configurated map as in Jupyter widget
Actuals
Colors and filters are not configured. Size and position of map is sent along, so if I store it looking at an empty area, when I open the html file it looks at the same field
In the Jupyter user guide for kepler.gl under the save section
# this will save current map
map_1.save_to_html(file_name='first_map.html')
# this will save map with provided data and config
map_1.save_to_html(data={'data_1': df}, config=config, file_name='first_map.html')
# this will save map with the interaction panel disabled
map_1.save_to_html(file_name='first_map.html', read_only=True)
So it looks like its a bug if the configuration parameter doesn't work or you are making the changes to the map configure after you set it equal to config. This would be fixed if you set
map_1.save_to_html(data={'data_1': df},
file_name='privateers.html',config=map_1.config)
I think it is a bug (or feature?) happens when you use the same cell to save the map configuration or still not print the map out yet. Generally, the config only exists after you really print the map out.
the problem, as far as I see it and how I solved at a similar problem is, that you 1) named your 'data' key in instancing the map different than you told it to save in the HTML 2).
map_1 = KeplerGl(height=500, data={'data': df},config=config)
map_1.save_to_html(data={'data_1': df}, file_name='privateers.html',config=config)
Name both keys the same and your HTML file will use the correct configuration.
Had this issue as well. Solved it by converting all pandas column dtypes to those that are json serializable: i.e. converting 'datetime' column from dtype <m8[ns] to object.

TCPDF On Next Page Header Goes Downwards

i am using TCPDF to generate pdf report. But i am having a wired issue, on first page header starts from right/correct y position but from 2nd page onward it goes downwards...
here is my code i am using:
<?php
require_once('./../../../Classes/tcpdf/config/tcpdf_config.php');
require_once('./../../../Classes/tcpdf/tcpdf.php');
class MYPDF extends TCPDF
{
//Page header
public function Header()
{
// Logo
$this->Image('./../../../images/bdGovt-Logo.gif', 5, 8, 25, '', 'GIF', '', 'T', false, 300, '', false, false, 0, false, false, false);
$this->Image('./../../../images/SEQAEP-Logo.gif', 275, 8, 15, '', 'GIF', '', 'T', false, 300, '', false, false, 0, false, false, false);
// Set font
$this->SetFont('helvetica', 'B', 12);
// Title
$this->ln();
$this->Cell(0, 0, 'Government Of People\'s Republic Of Bangladesh', 0, false, 'C', 0, '', 0, false, 'M', 'M');
$this->ln();
$this->Cell(0, 0, 'Secondary Education Quality & Access Enhancement Project (SEQAEP)', 0, false, 'C', 0, '', 0, false, 'M', 'M');
$this->ln();
$this->Cell(0, 0, 'Award Confirmation Form For Stipend And Tution', 0, false, 'C', 0, '', 0, false, 'M', 'M');
$this->ln();
$this->SetFont('', 'B', 10);
$this->Cell(0, 0, 'Semester: Jan - Jun 2015', 0, false, 'C', 0, '', 0, false, 'M', 'M');
$this->ln();
}
// Page footer
public function Footer()
{
// Position at 15 mm from bottom
$this->SetY(-20);
// Set font
$this->SetFont('helvetica', '', 8);
$this->MultiCell(93, 0, str_repeat('_', 45) . "\r\n" . 'Signature of Head of the Institute' . "\r\n" . 'Include Name with Official Seal', 0, 'C', 0, 0);
$this->MultiCell(94, 0, str_repeat('_', 45) . "\r\n" . 'Signature of Upazila Secondary Education Officer' . "\r\n" . 'Include Name with Official Seal', 0, 'C', 0, 0);
$this->MultiCell(93, 0, str_repeat('_', 45) . "\r\n" . 'Signature of Bank Officer' . "\r\n" . 'Include Name with Official Seal', 0, 'C', 0, 0);
$this->ln();
// Page number
$this->Cell(0, 10, 'Page ' . $this->getAliasNumPage() .' of ' . $this->getAliasNbPages(), 0, false, 'R', 0, '', 0, false, 'T', 'M');
}
}
$pdf = new MYPDF('P', PDF_UNIT, 'A3', true, 'UTF-8', false);
// set document information
$pdf->SetCreator('Mobile: +8801717219006');
$pdf->SetAuthor('Mohammad Kamrul Hassan');
$pdf->SetTitle('Siam Computer');
$pdf->SetSubject('Email: support#siamcomputer-bd.com');
$pdf->SetKeywords('www.siamcomputer-bd.com');
// set default header data
$pdf->SetHeaderData(PDF_HEADER_LOGO, PDF_HEADER_LOGO_WIDTH, PDF_HEADER_TITLE, PDF_HEADER_STRING);
// set header and footer fonts
$pdf->setHeaderFont(Array(PDF_FONT_NAME_MAIN, '', PDF_FONT_SIZE_MAIN));
$pdf->setFooterFont(Array(PDF_FONT_NAME_DATA, '', PDF_FONT_SIZE_DATA));
// set default monospaced font
$pdf->SetDefaultMonospacedFont(PDF_FONT_MONOSPACED);
// set margins
$pdf->SetMargins(10, 27, 10);
$pdf->SetHeaderMargin(PDF_MARGIN_HEADER);
$pdf->SetFooterMargin(PDF_MARGIN_FOOTER);
// set auto page breaks
$pdf->SetAutoPageBreak(TRUE, PDF_MARGIN_BOTTOM);
// set image scale factor
$pdf->setImageScale(PDF_IMAGE_SCALE_RATIO);
// set font
$pdf->SetFont('helvetica', 'B', 10);
// set cell padding
$pdf->setCellPaddings(1.000125, 1.000125, 1.000125, 1.000125);
// add a page
$pdf->AddPage();
$pdf->SetFont('helvetica', '', 8);
$strSInfoName = array('Location Code:', 'Bank Branch Code:', 'Bank Branch Name:', 'Division Name:', 'District Name:', 'Upazila Name:', 'Institute ID:', 'Institute Name:', 'EIIN No:');
$strSInfoValue = array('20902', '05229', 'GUNAGARI', 'CHITTAGONG', 'CHITTAGONG', 'BANSHKHALI', '01000400', 'MONAYEM SHAH AULIA HIGH SCHOOL', '104054');
$pdf->ln();
//file_put_contents('./wtf.txt', $pdf->GetY());
foreach($strSInfoName as $key => $value)
{
$pdf->Cell(30, 5, $strSInfoName[$key], 0, 0, 'L', 0);
$pdf->Cell(75, 5, $strSInfoValue[$key], 0, 0, 'L', 0);
$pdf->ln();
}
$pdf->SetFont('helvetica', 'B', 8);
$pdf->SetY(27);
$pdf->SetX(165);
$pdf->Cell(125, 0, 'Institution Summary', 1, 0, 'C', 0);
$pdf->ln();
$pdf->SetX(165);
$pdf->Cell(15, 11, 'Class', 1, 0, 'C', 0);
$pdf->Cell(20, 11, 'Students', 1, 0, 'C', 0);
$pdf->Cell(90, 0, 'Bellow Figures Showed In Taka', 1, 0, 'C', 0);
$pdf->ln();
$pdf->SetX(165);
$pdf->Cell(35, 0, '', 0, 0, 'C', 0);
$pdf->Cell(20, 0, 'Stipend', 1, 0, 'C', 0);
$pdf->Cell(20, 0, 'Tution', 1, 0, 'C', 0);
$pdf->Cell(30, 0, 'SSC Exam Fees', 1, 0, 'C', 0);
$pdf->Cell(20, 0, 'Total', 1, 0, 'C', 0);
$pdf->ln();
$pdf->SetFont('helvetica', '', 8);
$strDummyArray = array_fill(0, 6, '00');
$strDummyArray = array_fill(0, 6, $strDummyArray);
$strSizeArray = array(15, 20, 20, 20, 30, 20);
foreach($strDummyArray as $row)
{
$pdf->SetX(165);
foreach($row as $key=>$col)
{
$pdf->Cell($strSizeArray[$key], 0, $col, 1, 0, 'C', 0);
}
$pdf->ln();
}
$pdf->ln();
$pdf->ln();
$pdf->SetFont('helvetica', 'B', 8);
$pdf->Cell(10, 0, 'Class:', 0, 0, 'L', 0);
$pdf->SetFont('helvetica', '', 8);
$pdf->Cell(10, 0, '6', 0, 0, 'L', 0);
$pdf->ln();
$pdf->ln();
$pdf->Cell(7.5, 0, 'SL', 1, 0, 'C', 0);
$pdf->Cell(60, 0, 'Student Information', 1, 0, 'C', 0);
$pdf->Cell(15, 0, 'Stipened', 1, 0, 'C', 0);
$pdf->Cell(20, 0, 'Photo', 1, 0, 'C', 0);
$pdf->Cell(37.5, 0, 'Signature', 1, 0, 'C', 0);
$pdf->Cell(7.5, 0, 'SL', 1, 0, 'C', 0);
$pdf->Cell(60, 0, 'Student Information', 1, 0, 'C', 0);
$pdf->Cell(15, 0, 'Stipened', 1, 0, 'C', 0);
$pdf->Cell(20, 0, 'Photo', 1, 0, 'C', 0);
$pdf->Cell(37.5, 0, 'Signature', 1, 0, 'C', 0);
$pdf->ln();
$pdf->AddPage();
//Close and output PDF document
header('Content-Type: application/pdf');
header('Content-Disposition: inline; filename="test.pdf"', false);
$pdf->Output('test-' . time() . '.pdf', 'I');
?>
and this is how it looks on generated pdf (screenshots):
Page-1:
Page-2: (first line text (header) position get changed)
i want it start from same position on every page as first page
hope that makes sense? :)