How to write a Kusto query to select only the rows that have unique values in one field - azure-log-analytics

Having this input:
let t1 = datatable(id:string, col1:string, col2:string)
[
'1', 'col1_1', 'col2_1',
'2', 'col1_2', 'col2_2',
'3', 'col1_3', 'col2_3',
'4', 'col1_4', 'col2_4',
'1', 'col1_1', 'col2_11',
];
t1
| distinct id, col1
I need a query that will select only rows with unique values in "id" field. I understand that there are two possible outputs:
Output 1:
'1', 'col1_1', 'col2_1',
'2', 'col1_2', 'col2_2',
'3', 'col1_3', 'col2_3',
'4', 'col1_4', 'col2_4',
Output 2:
'2', 'col1_2', 'col2_2',
'3', 'col1_3', 'col2_3',
'4', 'col1_4', 'col2_4',
'1', 'col1_11', 'col2_11',


You can make use of any() aggregate function to pick up the col1 and col2 values based on unique values in 'id' column.
let t1 = datatable(id:string, col1:string, col2:string)
[
'1', 'col1_1', 'col2_1',
'2', 'col1_2', 'col2_2',
'3', 'col1_3', 'col2_3',
'4', 'col1_4', 'col2_4',
'1', 'col1_1', 'col2_11',
];
t1
| summarize any(col1), any(col2) by id


Would this work for your needs?
let t1 = datatable(id:string, col1:string, col2:string)
[
'1', 'col1_1', 'col2_1',
'2', 'col1_2', 'col2_2',
'3', 'col1_3', 'col2_3',
'4', 'col1_4', 'col2_4',
'1', 'col1_1', 'col2_11',
];
t1
| summarize col1 = make_set( col1 ), col2 = make_set( col2 ) by id

Related

Use Cusor instead of While Loop

I have a While Loop that goes through about 500,000 records and is very slow. I was reading online that I maybe able to use something called a Fast Forward Cursor to speed up the performance of the query. I have looked up fast forward cursors to try to apply it to my situation. Unfortunately, I have not been able to find anything that will explain how to convert a WHILE Loop into this type of cursor in the depth I need to be able to follow along and perform the necessary changes to my code. Any suggestions or advice would be appreciated. Below is the code I am using for my While Loop as well as some sample data.
The logic takes the test data which contains multiple months in one record and using the While Loop takes each record with multiple months and creates one record per month.
IF OBJECT_ID('tempdb.dbo.#TEST_Data') IS NOT NULL
DROP TABLE #TEST_Data
create table #TEST_Data (
Number varchar(50)
, P_Values varchar(50)
, StartDate date
, EndDate date
, NumberMonths numeric(7, 0)
, TotalAmount money
, MonthlyAmount money
, MultipleMonthid int
);
INSERT INTO #TEST_Data VALUES
('ABC1235', '1', '1/1/2018', '7/31/2018', '7', '-2066.82', '-295.26', '1')
, ('ABC1236', '1', '4/1/2019', '5/31/2019', '2', '431.2', '215.6', '2')
, ('ABC1237', '1', '4/1/2018', '5/31/2018', '2', '457.84', '228.92', '3')
, ('ABC1238', '1', '1/1/2019', '5/31/2019', '5', '-1279.4', '-255.88', '4')
, ('ABC1239', '1', '1/1/2018', '12/31/2018', '12', '569.52', '47.46', '5')
, ('ABC1240', '1', '2/1/2019', '5/31/2019', '4', '-546.08', '-136.52', '6')
, ('ABC1241', '1', '1/1/2019', '4/30/2019', '4', '149.4', '37.35', '7')
, ('ABC1242', '1', '1/1/2019', '5/31/2019', '5', '549.5', '109.9', '8')
, ('ABC1243', '1', '1/1/2019', '5/31/2019', '5', '1054.3', '210.86', '9')
, ('ABC1244', '1', '3/1/2018', '7/31/2018', '5', '2888.25', '577.65', '10')
, ('ABC1245', '1', '4/1/2019', '5/31/2019', '2', '379.5', '189.75', '11')
, ('ABC1246', '1', '5/1/2018', '12/31/2018', '8', '475.68', '59.46', '12')
, ('ABC1247', '1', '1/1/2019', '5/31/2019', '5', '-511.55', '-102.31', '13')
, ('ABC1248', '1', '1/1/2018', '7/31/2018', '7', '1531.6', '218.8', '14')
, ('ABC1249', '1', '1/1/2019', '5/31/2019', '5', '1043.2', '208.64', '15')
, ('ABC1250', '1', '1/1/2018', '7/31/2018', '7', '1554.35', '222.05', '16')
, ('ABC1251', '1', '6/1/2018', '7/31/2018', '2', '356.98', '178.49', '17')
, ('ABC1252', '1', '4/1/2018', '5/31/2018', '2', '356.98', '178.49', '18')
, ('ABC1253', '1', '1/1/2019', '5/31/2019', '5', '6787.5', '1357.5', '19')
, ('ABC1254', '1', '1/1/2019', '5/31/2019', '5', '-3327.2', '-665.44', '20')
, ('ABC1255', '1', '4/1/2018', '7/31/2018', '4', '-778.44', '-194.61', '21')
, ('ABC1256', '1', '1/1/2018', '7/31/2018', '7', '-2061.29', '-294.47', '22')
, ('ABC1257', '1', '1/1/2019', '5/31/2019', '5', '850.1', '170.02', '23')
, ('ABC1258', '1', '7/1/2018', '11/30/2018', '5', '3998.6', '799.72', '24')
, ('ABC1259', '1', '1/1/2018', '12/31/2018', '12', '9.48', '0.79', '25')
, ('ABC1260', '1', '10/1/2018', '11/30/2018', '2', '728.66', '364.33', '26')
, ('ABC1261', '1', '1/1/2018', '7/31/2018', '7', '4539.15', '648.45', '27')
, ('ABC1262', '1', '1/1/2019', '5/31/2019', '5', '-403.55', '-80.71', '28')
, ('ABC1263', '1', '1/1/2019', '5/31/2019', '5', '270.45', '54.09', '29')
, ('ABC1264', '1', '1/1/2018', '12/31/2018', '12', '2336.4', '194.7', '30')
, ('ABC1265', '1', '1/1/2018', '7/31/2018', '7', '1706.39', '243.77', '31')
, ('ABC1266', '1', '1/1/2018', '7/31/2018', '7', '-2055.48', '-293.64', '32')
, ('ABC1267', '1', '1/1/2018', '7/31/2018', '7', '1018.08', '145.44', '33')
, ('ABC1268', '1', '1/1/2018', '7/31/2018', '7', '-2782.08', '-397.44', '34')
, ('ABC1269', '1', '1/1/2018', '2/28/2018', '2', '-360.4', '-180.2', '35')
, ('ABC1270', '1', '1/1/2018', '7/31/2018', '7', '-968.66', '-138.38', '36')
, ('ABC1271', '1', '4/1/2019', '5/31/2019', '2', '-188.9', '-94.45', '37')
, ('ABC1272', '1', '1/1/2019', '5/31/2019', '5', '257.6', '51.52', '38')
, ('ABC1273', '1', '1/1/2019', '5/31/2019', '5', '1139.5', '227.9', '39')
, ('ABC1274', '1', '1/1/2019', '5/31/2019', '5', '-820', '-164', '40')
, ('ABC1275', '1', '1/1/2019', '5/31/2019', '5', '-1051.8', '-210.36', '41')
, ('ABC1276', '1', '2/1/2019', '3/31/2019', '2', '671.44', '335.72', '42')
, ('ABC1277', '1', '1/1/2019', '5/31/2019', '5', '1168.95', '233.79', '43')
, ('ABC1278', '1', '6/1/2018', '7/31/2018', '2', '479.5', '239.75', '44')
, ('ABC1279', '1', '4/1/2018', '5/31/2018', '2', '479.5', '239.75', '45')
, ('ABC1280', '1', '1/1/2018', '7/31/2018', '7', '1993.67', '284.81', '46')
, ('ABC1281', '1', '1/1/2018', '7/31/2018', '7', '7456.33', '1065.19', '47')
, ('ABC1282', '1', '1/1/2019', '5/31/2019', '5', '-1837.5', '-367.5', '48')
, ('ABC1283', '1', '6/1/2018', '7/31/2018', '2', '-312.14', '-156.07', '49')
, ('ABC1284', '1', '1/1/2018', '7/31/2018', '7', '781.62', '111.66', '50')
, ('ABC1285', '1', '1/1/2018', '6/30/2018', '6', '-3995.58', '-665.93', '51')
, ('ABC1286', '1', '1/1/2019', '5/31/2019', '5', '1159.15', '231.83', '52')
, ('ABC1287', '1', '1/1/2019', '5/31/2019', '5', '-9376.25', '-1875.25', '53')
, ('ABC1288', '1', '2/1/2018', '12/31/2018', '11', '1785.08', '162.28', '54')
, ('ABC1289', '1', '4/1/2018', '5/31/2018', '2', '-677.24', '-338.62', '55')
, ('ABC1290', '1', '1/1/2018', '12/31/2018', '12', '8.88', '0.74', '56')
, ('ABC1291', '1', '1/1/2018', '6/30/2018', '6', '-112.92', '-18.82', '57')
, ('ABC1292', '1', '6/1/2018', '7/31/2018', '2', '462.98', '231.49', '58')
, ('ABC1293', '1', '4/1/2018', '5/31/2018', '2', '-534.32', '-267.16', '59')
, ('ABC1294', '1', '2/1/2018', '3/31/2018', '2', '548.62', '274.31', '60')
, ('ABC1295', '1', '4/1/2018', '5/31/2018', '2', '286.42', '143.21', '61')
, ('ABC1296', '1', '1/1/2019', '5/31/2019', '5', '4907.15', '981.43', '62')
, ('ABC1297', '1', '4/1/2018', '12/31/2018', '9', '7.29', '0.81', '63')
, ('ABC1298', '1', '1/1/2019', '4/30/2019', '4', '2462.56', '615.64', '64')
, ('ABC1299', '1', '1/1/2018', '12/31/2018', '12', '8042.04', '670.17', '65')
, ('ABC1300', '1', '1/1/2019', '3/31/2019', '3', '1109.91', '369.97', '66')
, ('ABC1301', '1', '4/1/2018', '12/31/2018', '9', '2642.94', '293.66', '67')
, ('ABC1302', '1', '1/1/2018', '7/31/2018', '7', '-1430.45', '-204.35', '68')
, ('ABC1303', '1', '2/1/2018', '3/31/2018', '2', '1285.74', '642.87', '69')
, ('ABC1304', '1', '1/1/2019', '5/31/2019', '5', '-1163.45', '-232.69', '70')
, ('ABC1305', '1', '1/1/2018', '7/31/2018', '7', '2319.59', '331.37', '71')
, ('ABC1306', '1', '1/1/2018', '7/31/2018', '7', '6464.78', '923.54', '72')
, ('ABC1307', '1', '1/1/2018', '7/31/2018', '7', '4893.98', '699.14', '73')
, ('ABC1308', '1', '1/1/2019', '5/31/2019', '5', '-1249.35', '-249.87', '74')
, ('ABC1309', '1', '1/1/2018', '8/31/2018', '8', '2318', '289.75', '75')
, ('ABC1310', '1', '1/1/2018', '7/31/2018', '7', '2123.17', '303.31', '76')
, ('ABC1311', '1', '5/1/2018', '7/31/2018', '3', '-92.55', '-30.85', '77')
, ('ABC1312', '1', '1/1/2019', '5/31/2019', '5', '-789.9', '-157.98', '78')
, ('ABC1313', '1', '4/1/2018', '5/31/2018', '2', '4136.52', '2068.26', '79')
, ('ABC1314', '1', '1/1/2018', '7/31/2018', '7', '-450.59', '-64.37', '80')
, ('ABC1315', '1', '4/1/2018', '5/31/2018', '2', '-570', '-285', '81')
, ('ABC1316', '1', '1/1/2018', '12/31/2018', '12', '264.48', '22.04', '82')
, ('ABC1317', '1', '1/1/2018', '7/31/2018', '7', '1834.77', '262.11', '83')
, ('ABC1318', '1', '5/1/2018', '7/31/2018', '3', '1194.72', '398.24', '84')
, ('ABC1319', '1', '1/1/2018', '7/31/2018', '7', '-2398.27', '-342.61', '85')
, ('ABC1320', '1', '1/1/2018', '12/31/2018', '12', '2596.44', '216.37', '86')
, ('ABC1321', '1', '9/1/2018', '10/31/2018', '2', '1023.36', '511.68', '87')
, ('ABC1322', '1', '7/1/2019', '8/31/2019', '2', '-3772.86', '-1886.43', '88')
, ('ABC1323', '1', '1/1/2019', '5/31/2019', '5', '1223.6', '244.72', '89')
, ('ABC1324', '1', '1/1/2018', '7/31/2018', '7', '-1058.82', '-151.26', '90')
, ('ABC1325', '1', '1/1/2019', '4/30/2019', '4', '1669.16', '417.29', '91')
, ('ABC1326', '1', '7/1/2018', '12/31/2018', '6', '3031.5', '505.25', '92')
, ('ABC1327', '1', '8/1/2018', '12/31/2018', '5', '2325.1', '465.02', '93')
, ('ABC1328', '1', '1/1/2018', '7/31/2018', '7', '1571.22', '224.46', '94')
, ('ABC1329', '1', '1/1/2019', '5/31/2019', '5', '5521.05', '1104.21', '95')
, ('ABC1330', '1', '8/1/2019', '11/30/2019', '4', '26697.68', '6674.42', '96')
, ('ABC1331', '1', '1/1/2018', '12/31/2018', '12', '1254.96', '104.58', '97')
, ('ABC1332', '1', '1/1/2018', '7/31/2018', '7', '-1396.71', '-199.53', '98')
, ('ABC1333', '1', '6/1/2018', '7/31/2018', '2', '-1094.16', '-547.08', '99')
, ('ABC1334', '1', '4/1/2018', '5/31/2018', '2', '-1094.16', '-547.08', '100')
, ('ABC1335', '1', '1/1/2019', '5/31/2019', '5', '1900.6', '380.12', '101')
, ('ABC1336', '1', '3/1/2018', '7/31/2018', '5', '1013.75', '202.75', '102')
, ('ABC1337', '1', '5/1/2019', '6/30/2019', '2', '-1643.44', '-821.72', '103')
, ('ABC1338', '1', '9/1/2019', '10/31/2019', '2', '-5126.8', '-2563.4', '104')
, ('ABC1339', '1', '1/1/2018', '4/30/2018', '4', '1490.4', '372.6', '105')
, ('ABC1340', '1', '3/1/2018', '4/30/2018', '2', '1005.64', '502.82', '106')
, ('ABC1341', '1', '1/1/2018', '7/31/2018', '7', '-191.38', '-27.34', '107')
, ('ABC1342', '1', '2/1/2019', '3/31/2019', '2', '-376.08', '-188.04', '108')
, ('ABC1343', '1', '4/1/2019', '5/31/2019', '2', '647.96', '323.98', '109')
, ('ABC1344', '1', '1/1/2019', '5/31/2019', '5', '38.65', '7.73', '110')
, ('ABC1345', '1', '2/1/2018', '3/31/2018', '2', '1588.16', '794.08', '111')
, ('ABC1346', '1', '1/1/2019', '5/31/2019', '5', '322', '64.4', '112')
, ('ABC1347', '1', '1/1/2019', '5/31/2019', '5', '2396.75', '479.35', '113')
, ('ABC1348', '1', '4/1/2019', '5/31/2019', '2', '1403.52', '701.76', '114')
, ('ABC1349', '1', '1/1/2018', '7/31/2018', '7', '10790.43', '1541.49', '115')
, ('ABC1350', '1', '11/1/2018', '12/31/2018', '2', '437.62', '218.81', '116')
, ('ABC1351', '1', '1/1/2018', '10/31/2018', '10', '2188.1', '218.81', '117')
, ('ABC1352', '1', '1/1/2018', '3/31/2018', '3', '-663.72', '-221.24', '118')
, ('ABC1353', '1', '1/1/2019', '5/31/2019', '5', '1043.25', '208.65', '119')
, ('ABC1354', '1', '1/1/2018', '7/31/2018', '7', '287.21', '41.03', '120')
, ('ABC1355', '1', '1/1/2019', '5/31/2019', '5', '1979.2', '395.84', '121')
, ('ABC1356', '1', '9/1/2018', '11/30/2018', '3', '2.43', '0.81', '122')
, ('ABC1357', '1', '6/1/2018', '8/31/2018', '3', '2.43', '0.81', '123')
, ('ABC1358', '1', '1/1/2019', '5/31/2019', '5', '1167.8', '233.56', '124')
, ('ABC1359', '1', '1/1/2018', '4/30/2018', '4', '-23.36', '-5.84', '125')
, ('ABC1360', '1', '2/1/2018', '3/31/2018', '2', '325.62', '162.81', '126')
, ('ABC1361', '1', '1/1/2018', '7/31/2018', '7', '2151.31', '307.33', '127')
, ('ABC1362', '1', '1/1/2019', '5/31/2019', '5', '1770.25', '354.05', '128')
, ('ABC1363', '1', '1/1/2018', '7/31/2018', '7', '2207.66', '315.38', '129')
, ('ABC1364', '1', '4/1/2019', '5/31/2019', '2', '-8469.62', '-4234.81', '130')
, ('ABC1365', '1', '3/1/2019', '4/30/2019', '2', '192.76', '96.38', '131')
, ('ABC1366', '1', '4/1/2019', '5/31/2019', '2', '2217.02', '1108.51', '132')
, ('ABC1367', '1', '1/1/2019', '5/31/2019', '5', '-803.1', '-160.62', '133')
, ('ABC1368', '1', '2/1/2018', '7/31/2018', '6', '1203.66', '200.61', '134')
, ('ABC1369', '1', '1/1/2018', '7/31/2018', '7', '-732.13', '-104.59', '135')
, ('ABC1370', '1', '1/1/2019', '5/31/2019', '5', '-867.25', '-173.45', '136')
, ('ABC1371', '1', '1/1/2018', '12/31/2018', '12', '2616.24', '218.02', '137')
, ('ABC1372', '1', '1/1/2019', '5/31/2019', '5', '819.95', '163.99', '138')
, ('ABC1373', '1', '1/1/2018', '10/31/2018', '10', '1664.7', '166.47', '139')
, ('ABC1374', '1', '1/1/2019', '5/31/2019', '5', '8251.55', '1650.31', '140')
, ('ABC1375', '1', '3/1/2019', '5/31/2019', '3', '1712.97', '570.99', '141')
, ('ABC1376', '1', '4/1/2019', '5/31/2019', '2', '235.06', '117.53', '142')
, ('ABC1377', '1', '8/1/2018', '12/31/2018', '5', '393', '78.6', '143')
, ('ABC1378', '1', '1/1/2018', '7/31/2018', '7', '4555.6', '650.8', '144')
, ('ABC1379', '1', '1/1/2019', '5/31/2019', '5', '3224.15', '644.83', '145')
, ('ABC1380', '1', '1/1/2018', '5/31/2018', '5', '5438.65', '1087.73', '146')
, ('ABC1381', '1', '3/1/2018', '7/31/2018', '5', '1536.65', '307.33', '147')
, ('ABC1382', '1', '5/1/2018', '7/31/2018', '3', '755.46', '251.82', '148')
, ('ABC1383', '1', '8/1/2018', '9/30/2018', '2', '-622.38', '-311.19', '149')
, ('ABC1384', '1', '1/1/2019', '5/31/2019', '5', '2921.1', '584.22', '150')
, ('ABC1385', '1', '1/1/2019', '5/31/2019', '5', '-7026.65', '-1405.33', '151')
, ('ABC1386', '1', '1/1/2018', '4/30/2018', '4', '5617.76', '1404.44', '152')
, ('ABC1387', '1', '10/1/2018', '12/31/2018', '3', '1065.48', '355.16', '153')
, ('ABC1388', '1', '5/1/2018', '9/30/2018', '5', '2642.2', '528.44', '154')
, ('ABC1389', '1', '1/1/2019', '5/31/2019', '5', '1652.9', '330.58', '155')
, ('ABC1390', '1', '1/1/2018', '7/31/2018', '7', '-4544.89', '-649.27', '156')
, ('ABC1391', '1', '1/1/2019', '5/31/2019', '5', '3323', '664.6', '157')
, ('ABC1392', '1', '1/1/2019', '3/31/2019', '3', '-1866.93', '-622.31', '158')
, ('ABC1393', '1', '1/1/2018', '7/31/2018', '7', '-3962.28', '-566.04', '159')
, ('ABC1394', '1', '4/1/2019', '5/31/2019', '2', '1253.62', '626.81', '160')
, ('ABC1395', '1', '2/1/2018', '3/31/2018', '2', '-168.06', '-84.03', '161')
, ('ABC1396', '1', '1/1/2018', '7/31/2018', '7', '-10632.65', '-1518.95', '162')
, ('ABC1397', '1', '4/1/2019', '5/31/2019', '2', '-387.3', '-193.65', '163')
, ('ABC1398', '1', '1/1/2019', '3/31/2019', '3', '4618.59', '1539.53', '164')
, ('ABC1399', '1', '1/1/2018', '7/31/2018', '7', '-4465.93', '-637.99', '165')
, ('ABC1400', '1', '3/1/2019', '5/31/2019', '3', '819.15', '273.05', '166')
, ('ABC1401', '1', '1/1/2018', '7/31/2018', '7', '3840.83', '548.69', '167')
, ('ABC1402', '1', '1/1/2018', '7/31/2018', '7', '1059.24', '151.32', '168')
, ('ABC1403', '1', '1/1/2019', '3/31/2019', '3', '2261.64', '753.88', '169')
, ('ABC1404', '1', '1/1/2018', '7/31/2018', '7', '3103.1', '443.3', '170')
, ('ABC1405', '1', '5/1/2018', '12/31/2018', '8', '830.24', '103.78', '171')
, ('ABC1406', '1', '1/1/2019', '5/31/2019', '5', '1210.65', '242.13', '172')
, ('ABC1407', '1', '1/1/2019', '5/31/2019', '5', '-6883.3', '-1376.66', '173')
, ('ABC1408', '1', '2/1/2019', '5/31/2019', '4', '-398.36', '-99.59', '174')
, ('ABC1409', '1', '1/1/2019', '5/31/2019', '5', '-2524.4', '-504.88', '175')
, ('ABC1410', '1', '5/1/2018', '7/31/2018', '3', '-2734.62', '-911.54', '176')
, ('ABC1411', '1', '4/1/2018', '7/31/2018', '4', '-894.6', '-223.65', '177')
, ('ABC1412', '1', '1/1/2018', '3/31/2018', '3', '325.8', '108.6', '178')
, ('ABC1413', '1', '1/1/2019', '5/31/2019', '5', '871.55', '174.31', '179')
, ('ABC1414', '1', '4/1/2019', '5/31/2019', '2', '880.96', '440.48', '180')
, ('ABC1415', '1', '1/1/2018', '7/31/2018', '7', '1989.33', '284.19', '181')
, ('ABC1416', '1', '1/1/2018', '12/31/2018', '12', '19911.84', '1659.32', '182')
, ('ABC1417', '1', '1/1/2018', '7/31/2018', '7', '-1193.85', '-170.55', '183')
, ('ABC1418', '1', '1/1/2019', '3/31/2019', '3', '1215.84', '405.28', '184')
, ('ABC1419', '1', '4/1/2018', '5/31/2018', '2', '-2399.12', '-1199.56', '185')
, ('ABC1420', '1', '1/1/2018', '7/31/2018', '7', '521.71', '74.53', '186')
, ('ABC1421', '1', '1/1/2019', '5/31/2019', '5', '-867.25', '-173.45', '187')
, ('ABC1422', '1', '1/1/2018', '7/31/2018', '7', '8053.36', '1150.48', '188')
, ('ABC1423', '1', '2/1/2018', '7/31/2018', '6', '1124.76', '187.46', '189')
, ('ABC1424', '1', '1/1/2019', '2/28/2019', '2', '24.2', '12.1', '190')
, ('ABC1425', '1', '1/1/2018', '7/31/2018', '7', '2780.26', '397.18', '191')
, ('ABC1426', '1', '1/1/2018', '7/31/2018', '7', '-236.53', '-33.79', '192')
, ('ABC1427', '1', '4/1/2019', '5/31/2019', '2', '1342.9', '671.45', '193')
, ('ABC1428', '1', '7/1/2018', '11/30/2018', '5', '2276.85', '455.37', '194')
, ('ABC1429', '1', '1/1/2019', '5/31/2019', '5', '2096.25', '419.25', '195')
, ('ABC1430', '1', '1/1/2018', '2/28/2018', '2', '317', '158.5', '196')
, ('ABC1431', '1', '1/1/2019', '5/31/2019', '5', '-789.9', '-157.98', '197')
, ('ABC1432', '1', '1/1/2019', '5/31/2019', '5', '-4065.65', '-813.13', '198')
, ('ABC1433', '1', '1/1/2018', '7/31/2018', '7', '-1058.82', '-151.26', '199')
, ('ABC1434', '1', '1/1/2018', '7/31/2018', '7', '1148.84', '164.12', '200')
declare #start_date date, #end_date date, #Number varchar(50), #P_Values varchar(50), #Counter int, #id int, #end_id int, #MonthlyAmount money;
set #Counter = 1
select #end_id = max(MultipleMonthid) from #TEST_Data;
declare #months table (
id int identity (1,1)
, Number varchar(50)
, P_Values varchar(50)
, month_id int
, month_start date
, month_end date
, MultipleMonthid int
, MonthlyAmount money
, primary key (id)
);
while #Counter <= #end_id
BEGIN
SELECT #start_date = StartDate
,#end_date = EndDate
,#Number = Number
,#P_Values = P_Values
,#id = MultipleMonthid
,#MonthlyAmount = MonthlyAmount
FROM #TEST_Data
Where MultipleMonthid = #Counter
while #start_date <= #end_date
begin
insert into #months (Number, P_Values, month_id, month_start, month_end, MultipleMonthid, MonthlyAmount)
values (#Number, #P_Values,left(convert(varchar, #start_date, 112), 6), #start_date, eomonth(#start_date), #id, #MonthlyAmount);
set #start_date = dateadd(month, 1, #start_date)
end
set #Counter = #Counter + 1
end
;
select *
from #months

The fastest cursor would be straight SQL.(ie by not using a row by row to method to load records, in other words the while loop logic)
Here from the look of the query, you wish to insert as many records as there are between the start_date and end_dates by [Number], Here is how i would do this using straight sql.
declare #months table (
id int identity (1,1)
, Number varchar(50)
, P_Values varchar(50)
, month_id int
, month_start date
, month_end date
, MultipleMonthid int
, MonthlyAmount money
, primary key (id)
);
with data
as (--This block would be able to generate 2552 month gaps...
select row_number() over(order by (select null))-1 as rnk
from master..spt_values
)
insert
into #months
( [Number]
, P_Values
, month_id
, month_start
, month_end
, MultipleMonthid
, MonthlyAmount
)
select td.[Number]
, td.P_Values
, left(convert(varchar, dateadd(month,rnk,td.startdate), 112), 6) as month_id
, dateadd(month,rnk,td.startdate) as month_start
, eomonth(dateadd(month,rnk,td.startdate)) as month_end
, dense_rank() over(order by td.[Number] ) as multiplemonthid
, td.monthlyamount as MonthlyAmount
from #TEST_Data td
join data d
on dateadd(month,rnk,td.startdate)<=td.EndDate;
select *
from #months;
--1019 rows..
Here is the dbfiddle link which you can make use to verify your results
https://dbfiddle.uk/?rdbms=sqlserver_2019&fiddle=80c1342607f09583c39bf209805d4c3e

How to apply str.split() on pandas column?

Using Simple Data:
df = pd.DataFrame({'ids': [0,1,2], 'value': ['2 4 10 0 14', '5 91 19 20 0', '1 1 1 2 44']})
I need to convert the column to array, so I use:
df.iloc[:,-1] = df.iloc[:,-1].apply(lambda x: str(x).split())
X = df.iloc[:, 1:]
X = np.array(X.values)
but the problem is the data is being nested and I just need a matrix (3,5). How to make this properly and fast for large data (avoid looping)?

As said in the comments by #anky, #ScottBoston. You can use string method split along with expand parameter and finally change to NumPy:
df.iloc[:, 1].str.split(expand=True).values
array([['2', '4', '10', '0', '14'],
['5', '91', '19', '20', '0'],
['1', '1', '1', '2', '44']], dtype=object)

Conditional grouping in pandas and transpose

With an input dataframe framed out of a given CSV, I need to transpose the data based on certain conditions. The groupby should be applied based on Key value.
For any value in the same 'Key' group, if the 'Type' is "T", these values should be written on "T" columns labelled as T1, T2, T3...and so on.
For any value in the same 'Key' group, if the 'Type' is "P" and 'Code' ends with "00" these values should be written on "U" columns labelled as U1, U2, U3...and so on.
For any value in the same 'Key' group, if the 'Type' is "P" and 'Code' doesn't end with "00" these values should be written on "P" columns labelled as P1, P2, P3...and so on.
There might be n number of values of type T & P for any Key value and the output columns for T & P should be updated accordingly
Input Dataframe:
df = pd.DataFrame({'Key': ['1', '1', '1', '1', '1', '2', '2', '2', '2', '2'],
'Value': ['T101', 'T102', 'P101', 'P102', 'P103', 'T201', 'T202', 'P201', 'P202', 'P203'],
'Type': ['T', 'T', 'P', 'P', 'P', 'T', 'T', 'P', 'P', 'P'],
'Code': ['0', '0', 'ABC00', 'TWY01', 'JTH02', '0', '0', 'OUJ00', 'LKE00', 'WDF45']
})
Expected Dataframe:
Can anyone suggest an effective solution for this case?

Here's a possible solution using pivot.
import pandas as pd
df = pd.DataFrame({'Key': ['1', '1', '1', '1', '1', '2', '2', '2', '2', '2'],
'Value': ['T101', 'T102', 'P101', 'P102', 'P103', 'T201', 'T202', 'P201', 'P202', 'P203'],
'Type': ['T', 'T', 'P', 'P', 'P', 'T', 'T', 'P', 'P', 'P'],
'Code': ['0', '0', 'ABC00', 'TWY01', 'JTH02', '0', '0', 'OUJ00', 'LKE00', 'WDF45']
})
# Set up the U label
df.loc[(df['Code'].apply(lambda x: x.endswith('00'))) & (df['Type'] == 'P'), 'Type'] = 'U'
# Type indexing by key by type
df = df.join(df.groupby(['Key','Type']).cumcount().rename('Tcount').to_frame() + 1)
df['Type'] = df['Type'] + df['Tcount'].astype('str')
# Pivot the table
pv =df.loc[:,['Key','Type','Value']].pivot(index='Key', columns='Type', values='Value')
>>>pv
Type P1 P2 T1 T2 U1 U2
Key
1 P102 P103 T101 T102 P101 NaN
2 P203 NaN T201 T202 P201 P202
cdf = df.loc[df['Code'] != '0', ['Key', 'Code']].groupby('Key')['Code'].apply(lambda x: ','.join(x))
>>>cdf
Key
1 ABC00,TWY01,JTH02
2 OUJ00,LKE00,WDF45
Name: Code, dtype: object
>>>pv.join(cdf)
P1 P2 T1 T2 U1 U2 Code
Key
1 P102 P103 T101 T102 P101 None ABC00,TWY01,JTH02
2 P203 None T201 T202 P201 P202 OUJ00,LKE00,WDF45

How do I use count and group by as a condition for another table

So to preface, I'm a first-year comp sci student and we've only just started on SQL, so forgive me if the solution seems obvious.
We were given a database for Zoo, which has tables for Animals, Keepers, and a link entity (if that's the right word) for care roles, connecting the two.
(Schema below)
CREATE TABLE Animal (ID VARCHAR(6) PRIMARY KEY, Name VARCHAR(10), Species
VARCHAR(20),
Age SMALLINT, Sex VARCHAR(1), Weight SMALLINT, F_ID VARCHAR(6), M_ID
VARCHAR(6));
CREATE TABLE Keeper (Staff_ID VARCHAR(6) PRIMARY KEY, Keeper_Name
VARCHAR(20), Specialisation VARCHAR(20));
CREATE TABLE Care_Role (ID VARCHAR(6), Staff_ID VARCHAR(6), Role
VARCHAR(10), PRIMARY KEY (ID, Staff_ID));
Now the task we've been given is to work out which Keepers have been caring for more than 10 animals of the same species using the following data:
INSERT INTO Animal VALUES
('11', 'Horace', 'Marmoset', 99, 'M', 5, '2','1'),
('12', 'sghgdht', 'Marmoset', 42, 'M', 3, '2','1'),
('13', 'xgnyn', 'Marmoset', 37, 'F', 3, '1','11'),
('14', 'sbfdfbng', 'Marmoset', 12, 'F', 3, '1','11'),
('15', 'fdghd', 'Marmoset', 12, 'M', 3, '1','11'),
('16', 'Fred', 'Marmoset', 6, 'M', 3, '15','1'),
('17', 'Mary', 'Marmoset', 3, 'F', 3, '8','14'),
('18', 'Jane', 'Marmoset', 5, 'F', 3, '7','13'),
('19', 'dfgjtjt', 'Marmoset', 5, 'M', 3, '16','17'),
('20', 'Eric', 'Marmoset', 5, 'M', 3, '12','13'),
('21', 'tukyufyu', 'Marmoset', 5, 'M', 3, '12','73'),
('31', 'hgndghmd', 'Giraffe', 99, 'M', 5, '201','1'),
('32', 'sghgdht', 'Giraffe', 42, 'M', 3, '201','1'),
('33', 'xgnyn', 'Giraffe', 37, 'F', 3, '111','1'),
('34', 'sbfdfbng', 'Giraffe', 12, 'F', 3, '111','1'),
('35', 'fdghd', 'Giraffe', 12, 'M', 3, '111','6'),
('36', 'Fred', 'Lion', 6, 'M', 3, '151','111'),
('37', 'Mary', 'Lion', 3, 'F', 3, '81','114'),
('38', 'Jane', 'Lion', 5, 'F', 3, '71','113'),
('39', 'Kingsly', 'Lion', 9, 'M', 3, '161','117'),
('40', 'Eric', 'Lion', 11, 'M', 3, '121','113'),
('41', 'tukyufyu', 'Lion', 2, 'M', 3, '121','173'),
('61', 'hgndghmd', 'Elephant', 6, 'F', 225, '201','111'),
('62', 'sghgdht', 'Elephant', 10, 'F', 230, '201','111'),
('63', 'xgnyn', 'Elephant', 5, 'F', 300, '111','121'),
('64', 'sbfdfbng', 'Elephant', 11, 'F', 173, '111','121'),
('65', 'fdghd', 'Elephant', 12, 'F', 231, '111','666'),
('66', 'Fred', 'Elephant', 17, 'F', 333, '151','147'),
('67', 'Mary', 'Elephant', 3, 'F', 272, '81','148'),
('68', 'Jane', 'Elephant', 8, 'F', 47, '71','136'),
('69', 'dfgjtjt', 'Elephant', 9, 'F', 131, '161','172'),
('70', 'Eric', 'Elephant', 10, 'F', 333, '121','136'),
('71', 'tukyufyu', 'Elephant', 7, 'M', 114, '121','731');
INSERT INTO Keeper VALUES
('1', 'Roger', 'tdfhuihiu'),
('2', 'Sidra', 'rgegegtnrty'),
('3', 'Amit', 'ergetetnt'),
('4', 'Lucia', 'dvojivhwivih');
INSERT INTO Care_Role VALUES
('32', '1', 'feeding'),
('32', '2', 'washing'),
('61', '1', 'feeding'),
('62', '1', 'feeding'),
('63', '1', 'feeding'),
('64', '1', 'feeding'),
('65', '1', 'feeding'),
('66', '1', 'feeding'),
('67', '1', 'feeding'),
('68', '1', 'feeding'),
('69', '1', 'feeding'),
('70', '1', 'feeding'),
('71', '1', 'feeding'),
('11', '4', 'feeding'),
('12', '4', 'feeding'),
('13', '4', 'feeding'),
('14', '4', 'feeding'),
('15', '4', 'feeding'),
('16', '4', 'feeding'),
('17', '4', 'feeding'),
('18', '4', 'feeding'),
('19', '4', 'feeding'),
('20', '4', 'feeding'),
('21', '4', 'feeding');
So far what I've managed to come up with is this:
SELECT Keeper.Keeper_Name, Animal.Species, COUNT(Animal.Species)
FROM Keeper
JOIN Care_Role
ON Keeper.Staff_ID = Care_Role.Staff_ID
JOIN Animal
ON Care_Role.ID = Animal.ID
GROUP BY Animal.Species
But this is returning more than just the name (which is what I want), as well as showing all the people who have looked after animals, rather than just those who have looked after 10 or more, I was wondering if anyone had any ideas on how to help with this? Many thanks!

Your query should be returning an error, because Keeper.Keeper_name is not in the GROUP BY. You have made a good attempt. A reasonable way to start the query is:
SELECT k.Keeper_Name, a.Species, COUNT(*)
FROM Keeper k JOIN
Care_Role cr
ON k.Staff_ID = cr.Staff_ID JOIN
Animal a
ON cr.ID = a.ID
GROUP BY k.Keeper_Name, a.Species;
This will return the number of animals of a given species that each keeper cares for.
Note the following:
Table aliases are abbreviations for the table.
All column names are qualified.
This uses the shorthand of COUNT(*) instead of counting some particular column.
Your question adds an additional condition about 10 animals. You can fit that in using a HAVING clause.

how to avoid sub-query to gain performance

i have a reporting query which have 2 long sub-query
SELECT r1.code_centre, r1.libelle_centre, r1.id_equipe, r1.equipe, r1.id_file_attente,
r1.libelle_file_attente,r1.id_date, r1.tranche, r1.id_granularite_de_periode,r1.granularite,
r1.ContactsTraites, r1.ContactsenParcage, r1.ContactsenComm, r1.DureeTraitementContacts,
r1.DureeComm, r1.DureeParcage, r2.AgentsConnectes, r2.DureeConnexion, r2.DureeTraitementAgents,
r2.DureePostTraitement
FROM
( SELECT cc.id_centre_contact, cc.code_centre, cc.libelle_centre, a.id_equipe, a.equipe,
a.id_file_attente, f.libelle_file_attente, a.id_date, g.tranche, g.id_granularite_de_periode,
g.granularite, sum(Nb_Contacts_Traites) as ContactsTraites,
sum(Nb_Contacts_en_Parcage) as ContactsenParcage,
sum(Nb_Contacts_en_Communication) as ContactsenComm,
sum(Duree_Traitement/1000) as DureeTraitementContacts,
sum(Duree_Communication / 1000 + Duree_Conference / 1000 + Duree_Com_Interagent / 1000) as DureeComm,
sum(Duree_Parcage/1000) as DureeParcage
FROM agr_synthese_activite_media_fa_agent a, centre_contact cc,
direction_contact dc, granularite_de_periode g, media m, file_attente f
WHERE m.id_media = a.id_media
AND cc.id_centre_contact = a.id_centre_contact
AND a.id_direction_contact = dc.id_direction_contact
AND dc.direction_contact ='INCOMING'
AND a.id_file_attente = f.id_file_attente
AND m.media = 'PHONE'
AND ( ( g.valeur_min = date_format(a.id_date,'%d/%m') and g.granularite = 'Jour')
or ( g.granularite = 'Heure' and a.id_th_heure = g.id_granularite_de_periode) )
GROUP by cc.id_centre_contact, a.id_equipe, a.id_file_attente, a.id_date, g.tranche,
g.id_granularite_de_periode) r1,
(
(SELECT cc.id_centre_contact,cc.code_centre, cc.libelle_centre, a.id_equipe, a.equipe,
a.id_date, g.tranche, g.id_granularite_de_periode,g.granularite,
count(distinct a.id_agent) as AgentsConnectes,
sum(Duree_Connexion / 1000) as DureeConnexion,
sum(Duree_en_Traitement / 1000) as DureeTraitementAgents,
sum(Duree_en_PostTraitement / 1000) as DureePostTraitement
FROM activite_agent a, centre_contact cc, granularite_de_periode g
WHERE ( g.valeur_min = date_format(a.id_date,'%d/%m') and g.granularite = 'Jour')
AND cc.id_centre_contact = a.id_centre_contact
GROUP BY cc.id_centre_contact, a.id_equipe, a.id_date, g.tranche, g.id_granularite_de_periode )
UNION
(SELECT cc.id_centre_contact,cc.code_centre, cc.libelle_centre, a.id_equipe, a.equipe,
a.id_date, g.tranche, g.id_granularite_de_periode,g.granularite,
count(distinct a.id_agent) as AgentsConnectes,
sum(Duree_Connexion / 1000) as DureeConnexion,
sum(Duree_en_Traitement / 1000) as DureeTraitementAgents,
sum(Duree_en_PostTraitement / 1000) as DureePostTraitement
FROM activite_agent a, centre_contact cc, granularite_de_periode g
WHERE ( g.granularite = 'Heure'
AND a.id_th_heure = g.id_granularite_de_periode)
AND cc.id_centre_contact = a.id_centre_contact
GROUP BY cc.id_centre_contact,a.id_equipe, a.id_date, g.tranche, g.id_granularite_de_periode)
) r2
WHERE r1.id_centre_contact = r2.id_centre_contact
AND r1.id_equipe = r2.id_equipe AND r1.id_date = r2.id_date
AND r1.tranche = r2.tranche AND r1.id_granularite_de_periode = r2.id_granularite_de_periode
GROUP BY r1.id_centre_contact , r1.id_equipe, r1.id_file_attente,
r1.id_date, r1.tranche, r1.id_granularite_de_periode
ORDER BY r1.code_centre, r1.libelle_centre, r1.equipe,
r1.libelle_file_attente, r1.id_date, r1.id_granularite_de_periode,r1.tranche
the EXPLAIN shows
| id | select_type | table | type| possible_keys | key | key_len | ref| rows | Extra |
'1', 'PRIMARY', '<derived3>', 'ALL', NULL, NULL, NULL, NULL, '2520', 'Using temporary; Using filesort'
'1', 'PRIMARY', '<derived2>', 'ALL', NULL, NULL, NULL, NULL, '4378', 'Using where; Using join buffer'
'3', 'DERIVED', 'a', 'ALL', 'fk_Activite_Agent_centre_contact', NULL, NULL, NULL, '83433', 'Using temporary; Using filesort'
'3', 'DERIVED', 'g', 'ref', 'Index_granularite,Index_Valeur_min', 'Index_Valeur_min', '23', 'func', '1', 'Using where'
'3', 'DERIVED', 'cc', 'ALL', 'PRIMARY', NULL, NULL, NULL, '6', 'Using where; Using join buffer'
'4', 'UNION', 'g', 'ref', 'PRIMARY,Index_granularite', 'Index_granularite', '23', '', '24', 'Using where; Using temporary; Using filesort'
'4', 'UNION', 'a', 'ref', 'fk_Activite_Agent_centre_contact,fk_activite_agent_TH_heure', 'fk_activite_agent_TH_heure', '5', 'reporting_acd.g.Id_Granularite_de_periode', '2979', 'Using where'
'4', 'UNION', 'cc', 'ALL', 'PRIMARY', NULL, NULL, NULL, '6', 'Using where; Using join buffer'
NULL, 'UNION RESULT', '<union3,4>', 'ALL', NULL, NULL, NULL, NULL, NULL, ''
'2', 'DERIVED', 'g', 'range', 'PRIMARY,Index_granularite,Index_Valeur_min', 'Index_granularite', '23', NULL, '389', 'Using where; Using temporary; Using filesort'
'2', 'DERIVED', 'a', 'ALL', 'fk_agr_synthese_activite_media_fa_agent_centre_contact,fk_agr_synthese_activite_media_fa_agent_direction_contact,fk_agr_synthese_activite_media_fa_agent_file_attente,fk_agr_synthese_activite_media_fa_agent_media,fk_agr_synthese_activite_media_fa_agent_th_heure', NULL, NULL, NULL, '20903', 'Using where; Using join buffer'
'2', 'DERIVED', 'cc', 'eq_ref', 'PRIMARY', 'PRIMARY', '4', 'reporting_acd.a.Id_Centre_Contact', '1', ''
'2', 'DERIVED', 'f', 'eq_ref', 'PRIMARY', 'PRIMARY', '4', 'reporting_acd.a.Id_File_Attente', '1', ''
'2', 'DERIVED', 'dc', 'eq_ref', 'PRIMARY', 'PRIMARY', '4', 'reporting_acd.a.Id_Direction_Contact', '1', 'Using where'
'2', 'DERIVED', 'm', 'eq_ref', 'PRIMARY', 'PRIMARY', '4', 'reporting_acd.a.Id_Media', '1', 'Using where'
don't know it very clear, but i think is the problem of seems it take full scaning
than i change all the sub-query to views(create view as select sub-query), and the result is the same
thanks for any advice

Subquery and view will most of the time give you same result speed-wise.
If your subquery is not variabe, consider creating table that has same structure as your view, and occasionally do:
truncate table my_table;
insert into my_table select * from my_view;
...to cache your subquery data. If properly indexed, it will marginalize time lost on storing results of subquery in table, if data is not that frequently changed, or at least if you don't need up-to-date information on second-to-second basis.