Nairobi Stock Exchange Web Scraper

1
mkdir nse-stock-scraper && cd nse-stock-scraper

1
python3 -m venv venv
2
source venv/bin/activate

1
pip install scrapy pymongo[srv] dnspython python-dotenv beautifulsoup4
2
pip freeze > requirements.txt

1
scrapy startproject nse_scraper .

1
cd nse_scraper
2
scrapy genspider afx_scraper afx.kwayisi.org

1
from scrapy.settings.default_settings import CLOSESPIDER_PAGECOUNT, DEPTH_LIMIT
2
from scrapy.spiders import CrawlSpider, Rule
3
from bs4 import BeautifulSoup
4
from scrapy.linkextractors import LinkExtractor
5
6
7
class AfxScraperSpider(CrawlSpider):
8
   name = 'afx_scraper'
9
   allowed_domains = ['afx.kwayisi.org']
10
   start_urls = ['https://afx.kwayisi.org/nse/']
11
   user_agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'
12
   custom_settings = {
13
       DEPTH_LIMIT: 1,
14
       CLOSESPIDER_PAGECOUNT: 1
15
   }
16
17
   rules = (
18
       Rule(LinkExtractor(deny='.html', ), callback='parse_item', follow=False),
19
       Rule(callback='parse_item'),
20
   )
21
22
   def parse_item(self, response, **kwargs):
23
       print("Processing: " + response.url)
24
       # Extract data using css selectors
25
       row = response.css('table tbody tr ')
26
       # use XPath and regular expressions to extract stock name and price
27
       raw_ticker_symbol = row.xpath('td[1]').re('[A-Z].*')
28
       raw_stock_name = row.xpath('td[2]').re('[A-Z].*')
29
       raw_stock_price = row.xpath('td[4]').re('[0-9].*')
30
       raw_stock_change = row.xpath('td[5]').re('[0-9].*')
31
32
       # create a function to remove html tags from the returned list
33
       def clean_stock_symbol(raw_symbol):
34
           clean_symbol = BeautifulSoup(raw_symbol, "lxml").text
35
           clean_symbol = clean_symbol.split('>')
36
           if len(clean_symbol) > 1:
37
               return clean_symbol[1]
38
           else:
39
               return None
40
41
       def clean_stock_name(raw_name):
42
           clean_name = BeautifulSoup(raw_name, "lxml").text
43
           clean_name = clean_name.split('>')
44
           if len(clean_name[0]) > 2:
45
               return clean_name[0]
46
           else:
47
               return None
48
49
       def clean_stock_price(raw_price):
50
           clean_price = BeautifulSoup(raw_price, "lxml").text
51
           return clean_price
52
53
       # Use list comprehension to unpack required values
54
       stock_name = [clean_stock_name(r_name) for r_name in raw_stock_name]
55
       stock_price = [clean_stock_price(r_price) for r_price in raw_stock_price]
56
       ticker_symbol = [clean_stock_symbol(r_symbol) for r_symbol in raw_ticker_symbol]
57
       stock_change = [clean_stock_price(raw_change) for raw_change in raw_stock_change]
58
       if ticker_symbol is not None:
59
           cleaned_data = zip(ticker_symbol, stock_name, stock_price)
60
           for item in cleaned_data:
61
                scraped_data= {
62
                   'ticker_symbol': item[0],
63
                   'stock_name': item[1],
64
                   'stock_price': item[2],
65
                   'stock_change': stock_change }
66
       # yield info to scrapy
67
       yield scraped_data

1
from scrapy.item import Item, Field
2
3
4
class NseScraperItem(Item):
5
   # define the fields for your item here like:
6
   ticker_symbol = Field()
7
   stock_name = Field()
8
   stock_price = Field()
9
   stock_change = Field()

1
# pipelines.py
2
# useful for handling different item types with a single interface
3
import pymongo
4
from scrapy.exceptions import DropItem
5
6
from .items import NseScraperItem
7
8
9
class NseScraperPipeline:
10
   collection = "stock_data"
11
12
   def __init__(self, mongodb_uri, mongo_db):
13
       self.db = None
14
       self.client = None
15
       self.mongodb_uri = mongodb_uri
16
       self.mongo_db = mongo_db
17
       if not self.mongodb_uri:
18
           raise ValueError("MongoDB URI not set")
19
       if not self.mongo_db:
20
           raise ValueError("Mongo DB not set")
21
22
   @classmethod
23
   def from_crawler(cls, crawler):
24
       return cls(
25
           mongodb_uri=crawler.settings.get("MONGODB_URI"),
26
           mongo_db=crawler.settings.get('MONGO_DATABASE', 'nse_data')
27
       )
28
29
   def open_spider(self, spider):
30
       self.client = pymongo.MongoClient(self.mongodb_uri)
31
       self.db = self.client[self.mongo_db]
32
33
   def close_spider(self, spider):
34
       self.client.close()
35
  
36
   def clean_stock_data(self,item):
37
       if item['ticker_symbol'] is None:
38
           raise DropItem('Missing ticker symbol in %s' % item)
39
       elif item['stock_name'] == 'None':
40
           raise DropItem('Missing stock name in %s' % item)
41
       elif item['stock_price'] == 'None':
42
           raise DropItem('Missing stock price in %s' % item)
43
       else:
44
           return item
45
46
   def process_item(self, item, spider):
47
       """
48
       process item and store to database
49
       """
50
     
51
       clean_stock_data = self.clean_stock_data(item)
52
       data = dict(NseScraperItem(clean_stock_data))
53
       print(data)
54
       # print(self.db[self.collection].insert_one(data).inserted_id)
55
       self.db[self.collection].insert_one(data)
56
57
       return item

1
# settings.py
2
import os
3
from dotenv import load_dotenv
4
5
load_dotenv()
6
BOT_NAME = 'nse_scraper'
7
8
SPIDER_MODULES = ['nse_scraper.spiders']
9
NEWSPIDER_MODULE = 'nse_scraper.spiders'
10
11
# MONGODB SETTINGS
12
MONGODB_URI = os.getenv("MONGODB_URI")
13
MONGO_DATABASE = os.getenv("MONGO_DATABASE")
14
15
ITEM_PIPELINES = {
16
   'nse_scraper.pipelines.NseScraperPipeline': 300,
17
}
18
LOG_LEVEL = "INFO"
19
20
# USER_AGENT = 'nse_scraper (+http://www.yourdomain.com)'
21
22
# Obey robots.txt rules
23
ROBOTSTXT_OBEY = False
24
25
26
# Override the default request headers:
27
DEFAULT_REQUEST_HEADERS = {
28
 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
29
 'Accept-Language': 'en',
30
}
31
32
# Enable and configure HTTP caching (disabled by default)
33
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
34
HTTPCACHE_ENABLED = True
35
HTTPCACHE_EXPIRATION_SECS = 360
36
HTTPCACHE_DIR = 'httpcache'
37
# HTTPCACHE_IGNORE_HTTP_CODES = []
38
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

1
├ nse_stock_scraper
2
   ├ nse_scraper
3
       ├── __init__.py
4
       │   ├── items.py
5
       │   ├── middlewares.py
6
       │   ├── pipelines.py
7
       │   ├── settings.py
8
           ├─ stock_notification.py    
9
       │   └── spiders
10
       │       ├── __init__.py
11
       │       └── afx_scraper.py
12
   ├── README.md
13
   ├── LICENSE
14
   ├── requirements.txt
15
   └── scrapy.cfg
16
   ├── .gitignore
17
   ├── .env

1
source venv/bin/activate

1
MONGODB_URI=mongodb+srv://
2
MONGODB_DATABASE=
3
at_username=
4
at_api_key=
5
mobile_number=

1
mongodb+srv://<username>:<password>@<cluster-name>.mongodb.net/<database-name>?retryWrites=true&w=majority

1
scrapy crawl afx_scraper

1
pip install africastalking

1
# stock_notification.py
2
import africastalking as at
3
import os
4
from dotenv import load_dotenv
5
import pymongo
6
7
load_dotenv()
8
9
at_username = os.getenv("at_username")
10
at_api_key = os.getenv("at_api_key")
11
mobile_number = os.getenv("mobile_number")
12
mongo_uri = os.getenv("MONGODB_URI")
13
14
# Initialize the Africas sdk py passing the api key and username from the .env file
15
at.initialize(at_username, at_api_key)
16
sms = at.SMS
17
account = at.Application
18
19
20
ticker_data = []
21
22
# Create a function to send a message containing the stock ticker and price
23
def stock_notification(message: str, number: int):
24
   try:
25
       response = sms.send(message, [number])
26
       print(account.fetch_application_data())
27
       print(response)
28
   except Exception as e:
29
       print(f" Houston we have a problem: {e}")
30
31
# create a function to query mongodb for the stock price of Safaricom
32
def stock_query():
33
   client = pymongo.MongoClient(mongo_uri)
34
   db = client["nse_data"]
35
   collection = db["stock_data"]
36
   # print(collection.find_one())
37
   ticker_data = collection.find_one({"ticker": "BAT"})
38
   print(ticker_data)
39
   stock_name = ticker_data["name"]
40
   stock_price = ticker_data["price"]
41
   sms_data = { "stock_name": stock_name, "stock_price": stock_price }
42
   print(sms_data)
43
44
   message = f"Hello the current stock price of {stock_name} is {stock_price}"
45
   # check if Safaricom share price is more than Kes 39 and send a notification.
46
   if int(float(stock_price)) >= 38:
47
       # Call the function passing the message  and mobile_number as a arguments
48
       print(message)
49
       stock_notification(message, mobile_number)
50
   else:
51
       print("No notification sent")
52
  
53
   client.close()
54
55
   return sms_data
56
57
stock_query()

1
# afx_scraper.py
2
from nse_scraper.stock_notification import stock_query
3
4
# ...
5
6
# Add the following code to the end of the file
7
stock_query()

1
git init

1
touch .gitignore

1
# .gitignore
2
.env

1
git add .

1
git commit -m "Initial commit"

1
git remote add origin

1
git push -u origin master

1
touch .github/workflows/scraper-test.yml

1
name: Scraper test with MongoDB
2
3
on: [push]
4
5
jobs:
6
 build:
7
8
   runs-on: ubuntu-latest
9
   strategy:
10
     matrix:
11
       python-version: [3.8, 3.9, "3.10"]
12
       mongodb-version: ['4.4', '5.0', '6.0']
13
14
   steps:
15
   - uses: actions/checkout@v2
16
   - name: Set up Python ${{ matrix.python-version }}
17
     uses: actions/setup-python@v1
18
     with:
19
       python-version: ${{ matrix.python-version }}
20
   - name: Set up MongoDB ${{ matrix.mongodb-version }}
21
     uses: supercharge/mongodb-github-action@1.8.0
22
     with:
23
       mongodb-version: ${{ matrix.mongodb-version }}
24
   - name: Install dependencies
25
     run: |
26
       python -m pip install --upgrade pip
27
       pip install -r requirements.txt
28
   - name: Lint with flake8
29
     run: |
30
       pip install flake8
31
       # stop the build if there are Python syntax errors or undefined names
32
       flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
33
       # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
34
       flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
35
   - name: scraper-test
36
     run: |
37
       cd nse_scraper
38
       export MONGODB_URI=mongodb://localhost:27017
39
       export MONGO_DATABASE=nse_data
40
       scrapy crawl afx_scraper -a output_format=csv -a output_file=afx.csv
41
       scrapy crawl afx_scraper -a output_format=json -a output_file=afx.json

1
git add .
2
git commit -m "Add GitHub Actions"
3
git push