I already tried and found some posts related to the same question, but either they are not solved or their codes are somehow different.
I'm trying to scrape jobs from LinkedIn with the following piece of code, but some of the elements are not found in all posts, for example the seniority level, job function and job description.
i = 0
j = 0
print('Visiting the links and collecting information just started.') for i in range(len(links)):
try:
driver.get(links[i])
i=i+1
contents = driver.find_elements(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > section.top-card-layout.container-lined.overflow-hidden.babybear\:rounded-\[0px\] > div')
for content in contents:
try:
print(f'Scraping the Job Offer {j} P1')
job_titles.append(content.find_element(By.TAG_NAME,'h1').text)
company_names.append(content.find_element(By.CLASS_NAME,'topcard__flavor').text)
company_locations.append(content.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > section.top-card-layout.container-lined.overflow-hidden.babybear\:rounded-\[0px\] > div > div.top-card-layout__entity-info-container.flex.flex-wrap.papabear\:flex-nowrap > div > h4 > div:nth-child(1) > span.topcard__flavor.topcard__flavor--bullet').text)
post_dates.append(content.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > section.top-card-layout.container-lined.overflow-hidden.babybear\:rounded-\[0px\] > div > div.top-card-layout__entity-info-container.flex.flex-wrap.papabear\:flex-nowrap > div > h4 > div:nth-child(2) > span').text)
type_jobs = driver.find_elements(By.CLASS_NAME,'description__job-criteria-list')
for types in type_jobs:
if types.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > ul > li:nth-child(1) > span').text is not None:
seniority_level.append(types.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > ul > li:nth-child(1) > span').text)
else:
seniority_level.append('0')
if types.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > ul > li:nth-child(3) > span').text is not None:
job_function.append(types.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > ul > li:nth-child(3) > span').text)
else:
job_function.append('0')
time.sleep(2)
job_description = driver.find_elements(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > div')
for description in job_description:
time.sleep(2)
if description.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > div > section > div').text is not None:
job_desc.append(description.find_element(By.CSS_SELECTOR,'#main-content > section.core-rail.mx-auto.papabear\:w-core-rail-width.mamabear\:max-w-\[790px\].babybear\:max-w-\[790px\] > div > div > section.core-section-container.my-3.description > div > div > section > div').text)
else:
job_desc.append('0')
print(f'Scraping the Job Offer {j} P2 DONE')
time.sleep(2)
j+= 1
except:
pass
time.sleep(2)
except:
pass
The code works fine, it doesn't return any error if it doesn't find the element in one of the job posts, but the problem is that some list arrays are shorter than the others, so the data is mixed up as the values have different indexes. I already tried using some conditionals as above, but it doesn't work.
This is an example of a link not containing all elements: https://nl.linkedin.com/jobs/view/co%C3%B6rdinator-storingen-en-onderhoud-at-avia-weghorst-3450770711?refId=0Ja3RwR4I9%2BMnTYzR0B58A%3D%3D&trackingId=NBwBIMit1SA3VC3%2FAIy0vw%3D%3D&position=4&pageNum=0&trk=public_jobs_jserp-result_search-card
Related
I have written a code that will list out the scenarios which have a greater value from the given files. and I want to change the code as if the particular file is not present at the moment it should skip the block and move on to next block, but now the code shows an error if some of the files not present.
Here's my code:
Path ="/Users/xyz"
1regressions = pd.read_csv(Path+"/11regressions.csv")
1regressions = 1regressions[(1regressions['tp50_pct'] > 0.15) | (1regressions['tp90_pct'] > 0.15)]
1regressions['filename']='11regressions.csv'
1regressions = 1regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
2regressions = pd.read_csv(Path+"/12regressions.csv")
2regressions = 2regressions[(2regressions['tp50_pct'] > 0.15) | (2regressions['tp90_pct'] > 0.15)]
2regressions['filename']='12regressions.csv'
2regressions = 2regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
3regressions = pd.read_csv(Path+"/13regressions.csv")
3regressions = 3regressions[(3regressions['tp50_pct'] > 0.15) | (3regressions['tp90_pct'] > 0.15)]
3regressions['filename']='13regressions.csv'
3regressions = 3regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
4regressions = pd.read_csv(Path+"/14regressions.csv")
4regressions = 4regressions[(4regressions['tp50_pct'] > 0.15) | (4regressions['tp90_pct'] > 0.15)]
4regressions['filename']='14regressions.csv'
4regressions = 4regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
5regressions = pd.read_csv(Path+"/15regressions.csv")
5regressions = 5regressions[(5regressions['tp50_pct'] > 0.15) | (5regressions['tp90_pct'] > 0.15)]
5regressions['filename']='15regressions.csv'
5regressions = 5regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
5regressions = 5regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
6regressions = pd.read_csv(Path+"/16regressions.csv")
6regressions = 6regressions[(6regressions['tp50_pct'] > 0.15) | (6regressions['tp90_pct'] > 0.15)]
6regressions['filename']='16regressions.csv'
6regressions = 6regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
7regressions = pd.read_csv(Path+"/17regressions.csv")
7regressions = 7regressions[(7regressions['tp50_pct'] > 0.15) | (7regressions['tp90_pct'] > 0.15)]
7regressions['filename']='17regressions.csv'
7regressions = 7regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
8regressions = pd.read_csv(Path+"/18regressions.csv")
8regressions = 8regressions[(8regressions['tp50_pct'] > 0.15) | (JPH_regressions['tp90_pct'] > 0.15)]
8regressions['filename']='18regressions.csv'
8regressions = 8regressions[['filename','metric_type','content_format','tp50_pct', 'tp90_pct','mean_pct','iterations','perf_run_id','baseline_perf_run_id']]
result = 1regressions.append([2regressions, 3regressions,4regressions,5regressions,6regressions,7regressions,8regressions])
result.index = range(1, len(result) + 1)
result.to_csv(r'/Users/xyz/test.csv')
You can use os.path.exists.
But first, put all the repeated code in a loop.
dfs = []
Path =`.../xyz/'
for filename in list_with_filenames:
if os.path.exists(Path + filename): # this is what you were missing
df = pd.read_ csv(Path + filename)
# modify df
dfs.append(df)
dfs = pd.concat(dfs)
I Found a Binary Search Template here in leetcode
def binarySearch(nums, target):
"""
:type nums: List[int]
:type target: int
:rtype: int
"""
if len(nums) == 0:
return -1
left, right = 0, len(nums)
while left < right:
mid = (left + right) // 2
if nums[mid] == target:
return mid
elif nums[mid] < target:
left = mid + 1
else:
right = mid
# Post-processing:
# End Condition: left == right
if left != len(nums) and nums[left] == target:
return left
return -1
They say that "Template #2 is an advanced form of Binary Search. It is used to search for an element or condition which requires accessing the current index and its immediate right neighbor's index in the array."
I am struggling to understand what the information above means. In a usual binary search right would be
right = len(nums) - 1 # before while loop
right = mid - 1 # inside while loop
and the while loop would be
while left <= right:
You are performing a binary search on an array.
Lets say the array has 100 elements.
# left, right = 0, len(nums)
left = 0, right = 99
# mid = (left + right) // 2
mid = 49
# elif nums[mid] < target:
We need to move right.
# left = mid + 1
left = 50 and right = 99
# else:
We need to move left.
# right = mid
Left = 0, right = 49
I calculated three methods of the following with Numpy.
Avoiding the circle periodicity, I given the range is 0 to +180.
The calculation results of the three methods should match.
However, all calculation results are different.
Why is this?
degAry = []
sumDeg = 0
cosRad = 0
sinRad = 0
LEN = 300
RAD2DEG = 180.0 / PI # 57.2957795
for i in range(LEN):
deg = random.uniform(0,180)
rad = np.deg2rad(deg)
degAry.append(deg)
sumDeg += deg
cosRad += np.cos(rad)
sinRad += np.sin(rad)
print(np.arctan2( sinRad/LEN, cosRad/LEN ) * RAD2DEG) # 88.39325364335279
print(np.sum(degAry)/LEN) # 88.75448888951954
print(sumDeg/LEN) # 88.75448888951951
What makes you think that the mean angle and the angle of the mean vector should be the same? This is correct only for n = 1,2, for n = 3 degAry = [0, 90, 90] is easily verified to be a counter example: mean of the angles is 60 with tan = sqrt(3), mean vector is (1/3 2/3) corresponding to tan = 2.
EDIT
Mean of circular quantities
suggesting that the sin, cos approach is best.
Refactoring your code to use numpy exclusively. The two methods are different, however, the first two using RAD2DEG or the np.degrees yield the same results. The latter which used the sum of degrees divided by sample size differs.
It doesn't appear to be a summation issue (N=3000, sum in normal order, ascending then descending). They yield the same results
np.sum(deg) # 134364.25172174018
np.sum(np.sort(deg)) # 134364.25172174018
np.sum(np.sort(deg)[::-1]) # 134364.25172174018
I didn't carry it out with the summation for the cos and sin in radian form. I will leave that for others.
PI = np.pi
sumDeg = 0.
cosRad = 0.
sinRad = 0.
N = 30
RAD2DEG = 180.0 / PI # 57.2957795
deg = np.random.uniform(0, 90.0, N)
rad = np.deg2rad(deg)
sumDeg = np.sum(deg)
cosRad = np.sum(np.cos(rad))
sinRad = np.sum(np.sin(rad))
print(np.arctan2(sinRad/N, cosRad/N) * RAD2DEG)
print(np.degrees(np.arctan2(sinRad/N, cosRad/N)))
print(sumDeg/N)
Results for
> N = 1
> 22.746571717879792
> 22.746571717879792
> 22.746571717879792
>
> N= 30
> 48.99636699165551
> 48.99636699165551
> 49.000295118106884
>
> N = 300
> 44.39333460088003
> 44.39333460088003
> 44.44513528547155
>
> N = 3000
> 44.984167020219175
> 44.984167020219175
> 44.97574462726241
I have created this if and else if statements. Flat, Vertical, Facedown, Right Side, and Left Side work... However, I am not sure why the Upside down is not working. Here is my code:
checkOrientation= () => {
let orientation = "unknown";
if (z > .8 && Math.abs(x) < .2 && Math.abs(y) < .2) orientation = "flat";
else if (y > .9 && Math.abs(x) < .2 &&Math.abs(z) < .2) orientation = "vertical";
if (x < .1 && Math.abs(y) < .1 && Math.abs(z) < .3) orientation = "right side";
else if (z < .01 && Math.abs(x) > .9 && Math.abs(y) < .05) orientation = "left side";
if (x > .01 && Math.abs(y) < .7 && Math.abs(z) < .05) orientation = "upside down";
else if (x > .01 && Math.abs (y) < .01 && Math.abs(z) > .01) orientation = "face down";
this.setState({orientation});
}
I am using Shiny R, but I do not know to solve a problem in server. The "climate2" and "weather10" objects were not found. Likely, the "latlong" and "comp" objects show problems, but I do not know. The "latlong" argument of "forecast10day" function does not work with my "latlong" object and the "semi_join" function does not work with my "comp" object. I am copying only a part of the code because it is very long.
Many thanks!
Global R.
# Climate
climate <- cbind(tmin,tmax[,-c(1:3)],rhpm[,-c(1:3)],rham[,-c(1:3)])
names(climate) <- c("Long","Lat","Month","Tmin","Tmax","RHmin","RHmax")
head(climate,2)
Server:
latlong <- reactive({
latlong <- as.character(c(input$Lat,input$Long))
weather.10 <- forecast10day(set_location(lat_long = paste(latlong,collapse = ",")))
names(weather.10)[names(weather.10) == "temp_high"] <- "Tmax"
names(weather.10)[names(weather.10) == "temp_low"] <- "Tmin"
weather.10$Tmax <- fahrenheit.to.celsius(weather.10$Tmax, round = 0)
weather.10$Tmin <- fahrenheit.to.celsius(weather.10$Tmin, round = 0)
})
#Outputs
output$text0 <- renderText({
if(input$Irrigation =="No")
if(round(mean(min(weather.10$Tmin))) > 17 && round(mean(max(weather.10$Tmax))) < 35 && round(mean(min(weather.10$ave_humidity))) > 34 && round(mean(max(weather.10$ave_humidity))) < 87)
{
paste("The combination Biological and Chemical control is recomended in Drought or Rainy period.")
}else{
paste("Biological or chemical control or both may be inefficient and there are low risk of epidemics")
}
})
comp <- reactive ({
#Selecting options of user
comp <- data.frame(input$Long, input$Lat,input$Month)
climate <- semi_join(climate, comp, by=c("Long","Lat","Month"))
climate2 <- data.frame(unique(climate$Long),unique(climate$Lat),unique(climate$Month),round(mean(climate$Tmin)),
round(mean(climate$Tmax)),round(mean(climate$RHmin)),
round(mean(climate$RHmax)))
names(climate2) <- c("Long", "Lat","Mont","Tmin","Tmax","RHmin","RHmax")
})
output$text1 <- renderText({
if(input$Irrigation =="50-60%")
if(climate2$Tmax > 17 && climate2$Tmin < 35 && climate2$RHmin > 34 && climate2$RHmax < 87)
{
paste("The combination Biological and Chemical control is recomended.")
}else{
paste("Biological or chemical control or both may be inefficient. You can see more information in FORECASTING.")
}
})
This is my UI.
ui <- fluidPage(theme = shinytheme("superhero"),
h3("Information system to control Dry Root Rot in Common Beans"),
sidebarLayout(sidebarPanel(
numericInput("Long", label = h3("Longitude:"), value = -49),
numericInput("Lat", label = h3("Latitude:"), value = -17),
actionButton("recalc", "Show point"),
selectInput(inputId = "Irrigation",label = "Irrigation (Soil Available Water)",
choices = c("No","50-60%","80-90%","110-120%","140-150%"),
selected = "80-90%"
),
selectInput(inputId = "Month", label = "Current Month", choices = c("Jan","Feb","March","April","May","June","July",
"Aug","Sep","Oct","Nov","Dec")),