Scrapy-Selenium: Unable to get data - selenium

Question

I am trying to get some data using Scrapy-Selenium. The script is working fine except for the "URL", "Name of the share" and "Code of the share" fields. It is providing empty values for these fields. Though the "URL" field is containing a URL link, but it is not the full URL.
What am I actually missing here?
import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
from datetime import datetime
from scrapy import selector
from scrapy.selector import Selector
from markets_selenium.settings import *
class PriceSelSpider(scrapy.Spider):
name = 'price_sel'
def start_requests(self):
yield SeleniumRequest(
url="https://markets.ft.com",
wait_time=10,
callback=self.parse
)
def __init__(self):
chrome_options = Options()
chrome_options.add_argument('__headless')
chrome_path = SELENIUM_DRIVER_EXECUTABLE_PATH
driver = webdriver.Chrome(executable_path=chrome_path, options=chrome_options)
driver.set_window_size(1920, 1080)
driver.get('https://markets.ft.com/data/funds/tearsheet/historical?s=GB00B4NXY349:GBP')
time.sleep(2)
# self.html = driver.page_source
driver.quit()
def parse(self, response):
# resp = Selector(text=self.html)
now = datetime.now()
dt_string = now.strftime("%Y/%m/%d %H:%M:%S")
infos = response.xpath("//div[#class='mod-ui-table--freeze-pane__scroll-container']/table/tbody/tr")
for info in infos:
yield {
'Time': dt_string,
'URL': response.url,
'Name of the share': response.xpath(".//h1[#class='mod-tearsheet-overview__header__name mod-tearsheet-overview__header__name--large']/text()").get(),
'Code of the share': response.xpath(".//div[#class='mod-tearsheet-overview__header__symbol']/span/text()").get(),
'Date of share price ': info.xpath(".//td/span[1]/text()").get(),
'Opening price': info.xpath(".//td[2]/text()").get(),
'Highest price': info.xpath(".//td[3]/text()").get(),
'Lowest price': info.xpath(".//td[4]/text()").get(),
'Closing price': info.xpath(".//td[5]/text()").get(),
'Volume': info.xpath("(.//td/span)[3]/text()").get()
}
Output
{'Time': '2021/11/30 01:47:55', 'URL': 'https://markets.ft.com/data', 'Name of the share': None, 'Code of the share': None, 'Date of share price ': 'Friday, November 26, 2021', 'Opening price': '178.73', 'Highest price': '178.73', 'Lowest price': '178.73', 'Closing price': '178.73', 'Volume': '--'}
2021-11-30 01:47:55 [scrapy.core.scraper] DEBUG: Scraped from <200 https://markets.ft.com/data>