This question already has answers here:
pandas divide row value by aggregated sum with a condition set by other cell
(1 answer)
Pandas: normalize within the group
(3 answers)
calculate percentage based on specific column value
(1 answer)
Closed 12 months ago.
I have a long format df that has an amount column (absolute values) aggregated up to 3 different levels of date, country and group.
import pandas as pd
df = pd.DataFrame.from_dict([{ "date": "2022-02", "country": "Serbia", "group": 3, "amount": 33948 }, { "date": "2021-05", "country": "Thailand", "group": 3, "amount": 15857 }, { "date": "2021-05", "country": "Russia", "group": 2, "amount": 42855 }, { "date": "2021-11", "country": "Ukraine", "group": 3, "amount": 57306 }, { "date": "2021-06", "country": "Poland", "group": 2, "amount": 52898 }, { "date": "2022-02", "country": "Indonesia", "group": 3, "amount": 32330 }, { "date": "2021-11", "country": "Indonesia", "group": 1, "amount": 33791 }, { "date": "2022-02", "country": "China", "group": 3, "amount": 45050 }, { "date": "2021-12", "country": "Indonesia", "group": 1, "amount": 13865 }, { "date": "2022-03", "country": "Sweden", "group": 1, "amount": 45039 }, { "date": "2021-05", "country": "Colombia", "group": 3, "amount": 9363 }, { "date": "2022-01", "country": "Bangladesh", "group": 1, "amount": 47121 }, { "date": "2022-02", "country": "Indonesia", "group": 2, "amount": 18855 }, { "date": "2021-05", "country": "China", "group": 1, "amount": 49383 }, { "date": "2021-06", "country": "Turkmenistan", "group": 3, "amount": 61386 }, { "date": "2021-09", "country": "Kenya", "group": 3, "amount": 40434 }, { "date": "2022-03", "country": "Nicaragua", "group": 3, "amount": 3801 }, { "date": "2022-02", "country": "China", "group": 1, "amount": 39416 }, { "date": "2022-03", "country": "Brazil", "group": 1, "amount": 13657 }, { "date": "2021-05", "country": "Colombia", "group": 2, "amount": 23473 }, { "date": "2022-02", "country": "China", "group": 3, "amount": 19742 }, { "date": "2021-08", "country": "Russia", "group": 2, "amount": 45098 }, { "date": "2022-01", "country": "China", "group": 3, "amount": 15158 }, { "date": "2021-08", "country": "China", "group": 3, "amount": 18376 }, { "date": "2022-01", "country": "Slovenia", "group": 2, "amount": 71213 }, { "date": "2022-02", "country": "Czech Republic", "group": 2, "amount": 32744 }, { "date": "2021-06", "country": "Netherlands", "group": 1, "amount": 42706 }, { "date": "2021-07", "country": "China", "group": 2, "amount": 40277 }, { "date": "2021-08", "country": "United States", "group": 2, "amount": 3070 }, { "date": "2021-07", "country": "Germany", "group": 3, "amount": 17039 }, { "date": "2021-12", "country": "China", "group": 2, "amount": 8714 }, { "date": "2022-01", "country": "Malta", "group": 2, "amount": 44230 }, { "date": "2022-01", "country": "Russia", "group": 3, "amount": 33626 }, { "date": "2021-09", "country": "Greece", "group": 2, "amount": 72860 }, { "date": "2021-08", "country": "China", "group": 1, "amount": 59254 }, { "date": "2022-01", "country": "Japan", "group": 3, "amount": 18136 }, { "date": "2021-08", "country": "Venezuela", "group": 2, "amount": 14065 }, { "date": "2022-01", "country": "China", "group": 2, "amount": 36930 }, { "date": "2022-01", "country": "Honduras", "group": 2, "amount": 768 }, { "date": "2021-08", "country": "Vietnam", "group": 2, "amount": 33652 }, { "date": "2021-07", "country": "Ukraine", "group": 2, "amount": 54050 }, { "date": "2021-09", "country": "Indonesia", "group": 2, "amount": 50304 }, { "date": "2021-10", "country": "Peru", "group": 1, "amount": 27157 }, { "date": "2021-08", "country": "Brazil", "group": 3, "amount": 15869 }, { "date": "2021-11", "country": "Sweden", "group": 1, "amount": 32451 }, { "date": "2021-12", "country": "Mozambique", "group": 2, "amount": 29659 }, { "date": "2022-01", "country": "Argentina", "group": 2, "amount": 25282 }, { "date": "2021-06", "country": "Mongolia", "group": 2, "amount": 63027 }, { "date": "2021-07", "country": "Sudan", "group": 2, "amount": 5006 }, { "date": "2021-08", "country": "United States", "group": 2, "amount": 73414 }, { "date": "2021-05", "country": "China", "group": 3, "amount": 34759 }, { "date": "2021-12", "country": "Brazil", "group": 1, "amount": 636 }, { "date": "2021-06", "country": "Philippines", "group": 2, "amount": 59227 }, { "date": "2021-10", "country": "Russia", "group": 1, "amount": 28537 }, { "date": "2021-08", "country": "China", "group": 3, "amount": 23460 }, { "date": "2022-02", "country": "Philippines", "group": 2, "amount": 62968 }, { "date": "2021-10", "country": "Ukraine", "group": 3, "amount": 63908 }, { "date": "2021-10", "country": "Ukraine", "group": 3, "amount": 38263 }, { "date": "2021-06", "country": "Botswana", "group": 1, "amount": 15918 }, { "date": "2022-02", "country": "Russia", "group": 1, "amount": 31156 }, { "date": "2021-07", "country": "France", "group": 3, "amount": 64077 }, { "date": "2021-07", "country": "China", "group": 1, "amount": 18932 }, { "date": "2022-02", "country": "Russia", "group": 1, "amount": 45279 }, { "date": "2021-07", "country": "Russia", "group": 1, "amount": 7849 }, { "date": "2021-09", "country": "China", "group": 1, "amount": 52640 }, { "date": "2021-07", "country": "Peru", "group": 2, "amount": 19369 }, { "date": "2021-07", "country": "Greece", "group": 1, "amount": 20489 }, { "date": "2021-11", "country": "China", "group": 3, "amount": 30177 }, { "date": "2021-07", "country": "Portugal", "group": 1, "amount": 69521 }, { "date": "2021-06", "country": "Thailand", "group": 3, "amount": 17341 }, { "date": "2021-12", "country": "Peru", "group": 3, "amount": 27012 }, { "date": "2021-12", "country": "Afghanistan", "group": 1, "amount": 34146 }, { "date": "2021-11", "country": "Indonesia", "group": 1, "amount": 57619 }, { "date": "2021-05", "country": "Portugal", "group": 1, "amount": 37319 }, { "date": "2022-01", "country": "Denmark", "group": 1, "amount": 18370 }, { "date": "2022-01", "country": "United States", "group": 3, "amount": 4690 }, { "date": "2021-12", "country": "China", "group": 1, "amount": 35333 }, { "date": "2021-10", "country": "Indonesia", "group": 3, "amount": 74285 }, { "date": "2021-09", "country": "Mexico", "group": 1, "amount": 11260 }, { "date": "2021-11", "country": "Ukraine", "group": 3, "amount": 44389 }, { "date": "2021-11", "country": "France", "group": 3, "amount": 29432 }, { "date": "2021-08", "country": "Ecuador", "group": 1, "amount": 24529 }, { "date": "2021-08", "country": "Democratic Republic of the Congo", "group": 1, "amount": 5211 }, { "date": "2021-12", "country": "Georgia", "group": 3, "amount": 54164 }, { "date": "2021-05", "country": "France", "group": 2, "amount": 9046 }, { "date": "2021-05", "country": "Sweden", "group": 1, "amount": 10326 }, { "date": "2022-02", "country": "Madagascar", "group": 1, "amount": 70109 }, { "date": "2022-01", "country": "China", "group": 1, "amount": 25702 }, { "date": "2021-09", "country": "Poland", "group": 2, "amount": 46625 }, { "date": "2022-01", "country": "Czech Republic", "group": 1, "amount": 23806 }, { "date": "2021-06", "country": "Poland", "group": 2, "amount": 63310 }, { "date": "2021-11", "country": "Poland", "group": 3, "amount": 56290 }, { "date": "2021-12", "country": "Russia", "group": 3, "amount": 45846 }, { "date": "2021-09", "country": "Sweden", "group": 3, "amount": 26358 }, { "date": "2021-09", "country": "Colombia", "group": 2, "amount": 14682 }, { "date": "2021-11", "country": "China", "group": 1, "amount": 65021 }, { "date": "2022-02", "country": "Peru", "group": 1, "amount": 29406 }, { "date": "2022-01", "country": "China", "group": 1, "amount": 57333 }, { "date": "2021-05", "country": "Philippines", "group": 2, "amount": 28340 }, { "date": "2021-10", "country": "Japan", "group": 2, "amount": 37300 }])
example df
date country group amount
0 2022-02 Serbia 3 33948
1 2021-05 Thailand 3 15857
2 2021-05 Russia 2 42855
3 2021-11 Ukraine 3 57306
4 2021-06 Poland 2 52898
...
Date can be any yyyy-mm, Country can be any country, and group can be either 1, 2, 3.
What I would like to do is group by date and country, and then for each group, work out the relative percentage of the grouped date and country.
e.g. take the original df above to get something like (example of one date and one country):
date country group amount
2022-02 Serbia 1 33948
2 34567
3 96787
and then convert the amount to a percentage:
date country group amount_percentage
2022-02 Serbia 1 20.5
2 20.9
3 58.6
before collapsing back to the original format for all date and countries:
date country group amount_percentage
0 2022-02 Serbia 1 20.5
1 2022-02 Serbia 2 20.9
2 2022-02 Serbia 3 58.6
...
My current approach to tackling this is:
df.groupby(['date', 'country', 'group'])['amount'].sum().unstack()
which gives me grouped date and country columns, and the groups becoming column with relative amounts.
date country 1 2 3
2022-02 Serbia 33948 34567 96787
USA 23457 67589 23456
...
However I am not sure how to convert these to percentages of the row total, and then convert the dataframe back to final format.
How would you address this last part?
If I understand correctly, you can do a groupby and then transform('sum') on amount, and divide amount by that:
df['amount_percentage'] = df['amount'] / df.groupby(['date', 'country'], sort=False)['amount'].transform('sum') * 100
Output:
>>> df
date country group amount amount_percentage
0 2022-02 Serbia 3 33948 100.000000
1 2021-05 Thailand 3 15857 100.000000
2 2021-05 Russia 2 42855 100.000000
3 2021-11 Ukraine 3 57306 56.350853
4 2021-06 Poland 2 52898 45.520102
.. ... ... ... ... ...
95 2021-11 China 1 65021 68.300805
96 2022-02 Peru 1 29406 100.000000
97 2022-01 China 1 57333 42.430230
98 2021-05 Philippines 2 28340 100.000000
99 2021-10 Japan 2 37300 100.000000
I am developing a VB.NET application to interact with our new Shopify store. I am in the process of automating product additions/updates/removal based on data in our Oracle database.
Basically the application creates a new Shopify part for a pattern, then creates product variants for each sku in that pattern. It does this by POSTing and PUTing requests using the HttpWebRequest class. This was working great through yesterday afternoon, now today for some reason it's failing on the product variant creation/update with a 404 - Not Found error code.
The product add request JSON is below. This works perfectly, it creates the part on Shopify which I can then see through the admin panel. This is POSTed to the URL "https://key:password#bath-and-window-direct.myshopify.com/admin/products.json" (where key and password are replaced with our credentials)
{
"product": {
"id": 0,
"body_html": "A classic combination of embroidery and cut work form the flowing border, creating your own seaside retreat. ",
"title": "Seabreeze Sand",
"vendor": "SKL",
"product_type": "",
"published_scope": "global",
"tags": "J71227",
"variants": null,
"options": null,
"images": [{
"id": 0,
"product_id": 0,
"position": 1,
"src": "http:\/\/i320.photobucket.com\/albums\/nn353\/fkhphoto\/J71227main.jpg",
"variant_ids": null
}],
"image": null
}
}
This is the JSON reply I'm receiving after this call, and I can also see the part added in the Shopify admin panel:
{
"product": {
"id": 7874734983,
"title": "Seabreeze Sand",
"body_html": "A classic combination of embroidery and cut work form the flowing border, creating your own seaside retreat. ",
"vendor": "SKL",
"product_type": "",
"created_at": "2016-07-14T10:43:56-04:00",
"handle": "seabreeze-sand",
"updated_at": "2016-07-14T10:43:56-04:00",
"published_at": "2016-07-14T10:43:56-04:00",
"template_suffix": null,
"published_scope": "global",
"tags": "J71227",
"variants": [{
"id": 24925005383,
"product_id": 7874734983,
"title": "Default Title",
"price": "0.00",
"sku": "",
"position": 1,
"grams": 0,
"inventory_policy": "deny",
"compare_at_price": null,
"fulfillment_service": "manual",
"inventory_management": null,
"option1": "Default Title",
"option2": null,
"option3": null,
"created_at": "2016-07-14T10:43:56-04:00",
"updated_at": "2016-07-14T10:43:56-04:00",
"taxable": true,
"barcode": null,
"image_id": null,
"inventory_quantity": 1,
"weight": 0.0,
"weight_unit": "lb",
"old_inventory_quantity": 1,
"requires_shipping": true
}],
"options": [{
"id": 9404426823,
"product_id": 7874734983,
"name": "Title",
"position": 1,
"values": ["Default Title"]
}],
"images": [{
"id": 16242879303,
"product_id": 7874734983,
"position": 1,
"created_at": "2016-07-14T10:43:56-04:00",
"updated_at": "2016-07-14T10:43:56-04:00",
"src": "https://cdn.shopify.com/s/files/1/1363/2407/products/J71227main.jpg?v=1468507436",
"variant_ids": []
}],
"image": {
"id": 16242879303,
"product_id": 7874734983,
"position": 1,
"created_at": "2016-07-14T10:43:56-04:00",
"updated_at": "2016-07-14T10:43:56-04:00",
"src": "https://cdn.shopify.com/s/files/1/1363/2407/products/J71227main.jpg?v=1468507436",
"variant_ids": []
}
}
}
Immediately after that, I'm POSTing the following JSON to add a product variant to this newly created part, to the URL "https://key:password#bath-and-window-direct.myshopify.com/admin/products/7874734983/variants.json". This is the call that returns a 404 - Not Found error through the HttpWebRequest class.
{
"variant": {
"id": 0,
"product_id": 7874734983,
"title": "Seabreeze Tier Curtain in Sand",
"price": "11.99",
"sku": "J7122700013V09",
"compare_at_price": "0.00",
"position": 0,
"grams": 0,
"option1": "57X13 ROD POCKET VAL",
"option2": null,
"option3": null,
"taxable": true,
"barcode": "036326422417",
"weight": 0,
"weight_unit": "lb",
"inventory_quantity": 550,
"old_inventory_quantity": 550,
"requires_shipping": true,
"image_id": null,
"metafields": [{
"id": 0,
"key": "point1",
"value": "57 x 13 Valance",
"value_type": "string",
"namespace": "J7122700013V09"
},
{
"id": 0,
"key": "point2",
"value": "To achieve the look in the photo, use one valance and one tier pair.",
"value_type": "string",
"namespace": "J7122700013V09"
},
{
"id": 0,
"key": "point3",
"value": "Feels like you are in your own beach cottage.",
"value_type": "string",
"namespace": "J7122700013V09"
},
{
"id": 0,
"key": "point4",
"value": "100% Polyester.",
"value_type": "string",
"namespace": "J7122700013V09"
},
{
"id": 0,
"key": "point5",
"value": " ",
"value_type": "string",
"namespace": "J7122700013V09"
}]
}
}
What am I doing wrong? The URLs, when copied into a web browser, work perfectly fine...I can see all the JSON of the current data. But when attempting to add the variant, I'm getting the 404.
Try leaving out the "id": 0 properties — both from the variant and from the metafields. Shopify will determine the IDs automatically.
I am posting the following Order to the shopify api Order endpoint. The Order shows up in the shop and everything works as it should, except that the stock quantity of the variants in orders placed via the API are not decreased automatically by shopify. When I place an order within the admin console, they are decreased automatically. Shopify inventory tracking is turned on for the products. Any ideas would be greatly appreciated.
{
"order": {
"email": "someName#yahoo.com",
"financial_status": "paid",
"fulfillment_status": null,
"send_receipt": true,
"send_fulfillment_receipt": true,
"note": "Created by someName",
"line_items": [
{
"variant_id": 21718275463,
"quantity": 1,
"price": 99,
"requires_shipping": true,
"product_id": 6820646151
},
{
"variant_id": 21717700871,
"quantity": 1,
"price": 1000,
"requires_shipping": true,
"product_id": 6820646151
},
{
"variant_id": 21717690055,
"quantity": 1,
"price": 555,
"requires_shipping": true,
"product_id": 6821668807
}
],
"processing_method": "offsite",
"shipping_address": {
"first_name": "Chris",
"address1": "111 Love Road",
"phone": "9999999999",
"city": "St. Louis",
"zip": "63123",
"province": "MO",
"country": "United States",
"last_name": "Becker",
"name": "Chris Becker",
"country_code": "US",
"province_code": "MO"
},
"source_name": "someName",
"taxes_included": false,
"shipping_lines": [
{
"title": "standard",
"price": 0.00,
"code": null,
"source": "brand owner on shopify",
"carrier_identifier": null,
"tax_lines": null
}
],
"tags": "someName"
}
}
{
"variant": {
"id": 21718275463,
"product_id": 6820646151,
"title": "m / red",
"price": "99.00",
"sku": "",
"position": 2,
"grams": 0,
"inventory_policy": "deny",
"compare_at_price": "900.00",
"fulfillment_service": "manual",
"inventory_management": "shopify",
"option1": "m",
"option2": "red",
"option3": null,
"created_at": "2016-05-27T13:16:26-04:00",
"updated_at": "2016-05-28T13:28:20-04:00",
"taxable": false,
"barcode": "",
"image_id": 13217378823,
"inventory_quantity": 1,
"weight": 0,
"weight_unit": "lb",
"old_inventory_quantity": 1,
"requires_shipping": true
}
}
{
"variant": {
"id": 21717700871,
"product_id": 6820646151,
"title": "s / green",
"price": "1000.00",
"sku": "",
"position": 1,
"grams": 0,
"inventory_policy": "deny",
"compare_at_price": "1111.00",
"fulfillment_service": "manual",
"inventory_management": "shopify",
"option1": "s",
"option2": "green",
"option3": null,
"created_at": "2016-05-27T13:05:56-04:00",
"updated_at": "2016-05-28T12:17:22-04:00",
"taxable": true,
"barcode": "",
"image_id": 13160712135,
"inventory_quantity": 2,
"weight": 0,
"weight_unit": "lb",
"old_inventory_quantity": 2,
"requires_shipping": true
}
}
{
"variant": {
"id": 21717690055,
"product_id": 6821668807,
"title": "Default Title",
"price": "555.00",
"sku": "",
"position": 1,
"grams": 0,
"inventory_policy": "deny",
"compare_at_price": "666.00",
"fulfillment_service": "manual",
"inventory_management": "shopify",
"option1": "Default Title",
"option2": null,
"option3": null,
"created_at": "2016-05-27T13:05:39-04:00",
"updated_at": "2016-05-28T12:17:22-04:00",
"taxable": true,
"barcode": "",
"image_id": null,
"inventory_quantity": 2,
"weight": 0,
"weight_unit": "lb",
"old_inventory_quantity": 2,
"requires_shipping": true
}
}
You created a fake order using the API. Fake orders like that don't transact money or trigger the usual internal checks and balances without some extra effort. Maybe if you tried adding a fulfillment to the order, Shopify might ding inventory levels? Seems like something to try anyway.