Scrape online academic materials using python

 You know it can be a boring task to manually collect academic material you found online. In this blog post, I will demonstrate how I use python to collect some academic thesis, journals, and other materials for my profession.

Online Scientific Research Journals: 

Here my professor wants to have all the journals and their details published by "Scientific Research and Community Publishers" onlinescientificresearch.com neatly arranged in a spreadsheet table.

The specific details required are the journal name/title, the page URL, the description, cover image and ISSN number.

All the details should be organized in a spreadsheet as seen below.

The code:

import json
import requests
import pandas as pd
from bs4 import BeautifulSoup



# Section 1: Scrape journals page URLs and thumbnail images

url = 'https://www.onlinescientificresearch.com/journals.php'

# Get user-agent from: http://www.useragentstring.com/
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}

response = requests.get(url, headers=headers)
html = response.text

soup = BeautifulSoup(html, 'html.parser')
journals = soup.find_all("div", {'class':'col-12 col-sm-6 col-lg-3'})

print(len(journals))
# ---------------------------------------------------------------

# Section 2: Extract paths to journals URL and thumbnail image...

url_list = []
image_list = []

for j in journals:
    url = j.find('a')['href']
    img = j.find('img')['src']
    
    url_list.append(url)
    image_list.append(img)
    
print('Done...')


# ---------------------------------------------------------------
# Section 3: Create dataframe and construct other details...

df = pd.DataFrame([url_list, image_list]).T
df.columns = ['Journal URL', 'Journal IMAGE URL']
# -------------------------------------
####### Construct Journal Name #######
df['Journal Name'] = df['Journal URL'].apply(lambda row: row.split('/')[-1].replace('.php', '').replace('-', ' ').title())


####### Construct Journal Description #######
def get_journal_descr(url):
    # Get user-agent from: http://www.useragentstring.com/
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}

    response = requests.get(url, headers=headers)
    html = response.text
    
    soup = BeautifulSoup(html, 'html.parser')
    journal_descr = soup.find("div", {'class':'card-body'})
    
    return journal_descr.text
# -------------------------------------
# Scrape Journal description into a list 
j_descr_list = []
i = 1

for url in df['Journal URL']:
    print(i, 'Processing...', url)
    j_descr = get_journal_descr(url)
    
    j_descr_list.append((url, j_descr))
    i = i+1

desc_df = pd.DataFrame(j_descr_list)
# -------------------------------------

# We have to access each journal url page to get its description...
# df['Journal description'] = df['Journal URL'].apply(lambda url: get_journal_descr(url))
df['Journal description'] = desc_df[1]


####### Construct Journal ISSN #######
# We have to use OCR on the journal thumb nail to get its ISSN...
# Using OCR API at: https://ocr.space/ocrapi....

headers = {
    'apikey': 'helloworld', # 'helloworld'
    'content-type': 'application/x-www-form-urlencoded',
}

issn_list = []

for thumbnail in df['Journal IMAGE URL']:
    print('Processing....', thumbnail)
    
    data = f'isOverlayRequired=true&url={thumbnail}&language=eng'

    response = requests.post('https://api.ocr.space/Parse/Image', headers=headers, data=data, verify=False)

    result = json.loads(response.content.decode()) # Convert the result to dictionary using json.loads() function
    # type(result)

    # Check the dict keys, the ISSN is in: ParsedResults >> 0 >> ParsedText
    issn = result['ParsedResults'][0]['ParsedText'].strip().split('\r\n')[-1]

    issn_list.append(issn)

df['Journal ISSN'] = issn_list

df

Extracting the journal ISSN was definitely the trickiest part as it requires working with OCR API.

M.Sc. in GIST Theses

Master of Science (Geographic Information Science and Technology) Theses by University of Southern California. 

Here our professor wants the thesis details arranged in a table seen above.

Lets start by inspecting the html tags on the web page.

Here I copied the parent div tag that contains the needed data into a local html file. With this we don't need to send request to the website.

import pandas as pd
from bs4 import BeautifulSoup

# Copy the parent div tag into a html/txt file...
html_file = r"C:\Users\Yusuf_08039508010\Documents\Jupyter_Notebook\2022\M.S. IN GIST THESES\M.S. IN GIST THESES.HTML"

# Use BeautifulSoup to read the html div tag....
with open(html_file, encoding='utf-8') as f:
    div_data = f.read()

soup = BeautifulSoup(div_data, 'html.parser')

thesis_years = soup.find_all("h3")

thesis_authors = soup.find_all("strong")
thesis_authors = [ a.text for a in thesis_authors ]

thesis_topics = soup.find_all("em")
thesis_topics = [ t.text for t in thesis_topics ]

thesis_advisor = soup.find_all("p")
thesis_advisor = [ a.text for a in thesis_advisor if 'Advisor:' in a.text ]

thesis_pdf = soup.find_all("a")
thesis_pdf = [ link.get('href') for link in thesis_pdf if 'Abstract Text' not in link.text ]

# --------------------------------------------
df = pd.DataFrame(thesis_authors, columns=['Author'])
df['Topic'] = thesis_topics
df['Advisor'] = thesis_advisor
df['PDF Link'] = thesis_pdf

df

The code below will download the PDF files to local disc using the requests library.

i = 1
for indx, row in df.iterrows():
    link = row['PDF Link']
    print('Processsing...', link)

    pdf_name = str(i) +'_'+ link.split('/')[-1]
    pdf_file = requests.get(link, timeout=10).content

    with open( f'Thesis PDF\\{pdf_name}', 'wb' ) as f:
        f.write(pdf_file)
        
    i += 1
    # break


print('Finished...')

Journal - Nigerian Institution of Surveyors

This was little bit trick because the web page had inconsistent html tags.

import requests
import pandas as pd
from bs4 import BeautifulSoup


url = 'https://nisngr.net/journal/'
response = requests.get(url, verify=False)
html = response.text
# ----------------------------


soup = BeautifulSoup(html, 'html.parser')
div_boxes = soup.find_all("div", {'class':'wpb_text_column wpb_content_element'})
# ----------------------------


papers_dict = {}
for div in div_boxes:
    papers = div.find_all('a')
    
    for link in papers:
        papers_dict[link.text] = link['href']
# ----------------------------

df = pd.DataFrame([papers_dict]).T
df

Thank you for reading.

Automate boring tasks in QGIS with PyQGIS

 In this post, I will use PyQGIS to automate some boring tasks I often encounter in QGIS. Hope you will find something useful to your workflow. Lets get started...

If you don't know what pyqgis is, then read this definition by hatarilabs.com: "PyQGIS is the Python environment inside QGIS with a set of QGIS libraries plus the Python tools with the potential of running other powerful libraries as Pandas, Numpy or Scikit-learn".

PyQGIS allows users to automate workflow and extend QGIS with the use of Python libraries and the documentation can be accessed here.

This means knowledge of python programming is required to understand some of the codes below.

  Task 1~ Count number of opened/loaded layers in the layer panel

I often find myself trying to count the layers in my QGIS project layer panel, so a simple pyqis script to automate the process will be ideal especially when there are many layers on the layer panel to count.

# This will return the all layers on the layer panel
all_layers = QgsProject.instance().mapLayers().values()
print('There are', len(all_layers), 'on the layer panel.')

  Task 2~ Count features in loaded vector layer

In this task, I want to get the number of features in each layer am working on. This is similar to 'Show Feature Count' function when you right-click on a vector layer.

# Get all layers into a list....
all_layers = list(QgsProject.instance().mapLayers().values())

# Get all displayed names of layer and corresponding number of features ...
ftCounts = [ (l.name(), l.featureCount()) for l in all_layers ]
print(ftCounts)

  Task 3~ Switch on/off all layers

To turn ON or OFF all layer can be frustrating when you got many layers to click through. So why not auto mate it in just a click.

# Get list of layers from the layer's panel...
qgis_prjt_lyrs = QgsProject.instance().layerTreeRoot().findLayers()

# Use index to Set layer on or off....
qgis_prjt_lyrs[20].setItemVisibilityChecked(True) # True=On, False=Off

# Do for all...
for l in qgis_prjt_lyrs:
    l.setItemVisibilityChecked(False)

  Task 4~ Identify layers that are on/off

Lets extend task3 above, so we know which layers are on (visible) and which layers are off (hidden).

# Get list of layers from the layer's panel...
qgis_prjt_lyrs = QgsProject.instance().layerTreeRoot().findLayers()

# Check if a layer is visible or not...
layer_visibility_check = [ (l.name(), l.isVisible()) for l in qgis_prjt_lyrs ]
print(layer_visibility_check)

visibility_ture = [ l.name() for l in qgis_prjt_lyrs if l.isVisible() == True ]
print('Number of visible layers:', len(visibility_ture))

visibility_false = [ l.name() for l in qgis_prjt_lyrs if l.isVisible() == False ]
print('Number of visible layers:', len(visibility_false))

  Task 5~ Read file path of layers

This is useful when you have many layers and don't know where they are located on your machine. You will also see interesting paths to other remote layer such as WMS, etc

# Returns path to every layer...
layer_paths = [layer.source() for layer in QgsProject.instance().mapLayers().values()]
print(layer_paths)

  Task 6~ Read layer type of layers

We can check the 'type' of a layer.

# Get dict of layers from the layer's panel...
layersDict = QgsProject.instance().mapLayers()


for (id, map) in layersDict.items():
    print(map.name(), '>>', map.type())

  Task 7~ Create multiple attribute fields/columns

Lets say we want to add multiple integer fields/columns to a vector layer. The code below will create attribute fields for year 2000 to 2023, that is twenty three (23) attribute columns/fields on the selected vector layer.

# Get Layer by name...
layer = QgsProject.instance().mapLayersByName("NIG LGA")[0]

# Define dataProvider for layer
layer_provider = layer.dataProvider()

# Add an Integer attribute field and update fields...
layer_provider.addAttributes([QgsField("2000", QVariant.Int)])
layer.updateFields()

# Add bulk attribute fields...
for x in range(2001, 2023):
    layer_provider.addAttributes([QgsField(str(x), QVariant.Int)])
    layer.updateFields()

print('Done...')

  Task 8~ Read/List all names of layers on layer panel

Here we just want to return the displayed names of layers.

# Get all layers into a list....
all_layers = list(QgsProject.instance().mapLayers().values())

# Get all displayed names of layer
all_layers_names = [ l.name() for l in all_layers ]
print(all_layers_names)

  Task 9~ Save attribute table to dataframe

# Save attribute table into Dataframe...

import pandas as pd

# Get Layer by name...
layer = QgsProject.instance().mapLayersByName("NIG LGA")[0]

# get attribute columns names
col_names = [ field.name() for field in layer.fields() ]

lga_list = []
state_list = []
apc_list = []
pdp_list = []
lp_list = []
nnpp_list = []
winner_list = []


for feature in layer.getFeatures():
    lga_list.append(feature['lga_name'])
    state_list.append(feature['state_name'])
    apc_list.append(feature['APC'])
    pdp_list.append(feature['PDP'])
    lp_list.append(feature['LP'])
    nnpp_list.append(feature['NNPP'])
    winner_list.append(feature['Winner'])

df = pd.DataFrame([state_list, lga_list, apc_list, pdp_list, lp_list, nnpp_list, winner_list]).T

df.to_csv(r'C:\Users\Yusuf_08039508010\Desktop\...\test.csv')

print('Done....')

  Task 10~ Select from multiple layers and attribute fields

Here we want to conduct multiple selection of given keywords from all listed layers and all attribute fields.

# Query to Select from all listed layers and all attribute fields
search_for = {'Bauchi', 'SSZ', 'Edo', 'Yobe'}

for lyr in QgsProject.instance().mapLayers().values():
    if isinstance(lyr, QgsVectorLayer):
        to_select = []
        # fieldlist = [f.name() for f in lyr.fields()]
        for f in lyr.getFeatures():
            # Check if any of the search keyword intersects to
            # feature's row attribute. If true, get the feature ID for selection...
            if len(search_for.intersection(f.attributes())) > 0:
                to_select.append(f.id())
        if len(to_select) > 0:
            lyr.select(to_select)

  Task 11~ Convert multiple GeoJSON files to shapefiles

import glob

input_files = glob.glob(r'C:\Users\Yusuf_08039508010\Desktop\Working_Files\GIS Data\US Zip Codes\*.json')
for f in input_files:
    out_filename = f.split('\\')[-1].split('.')[0]
    input_file = QgsVectorLayer(f, "polygon", "ogr")
    
    if input_file.isValid() == True:
        QgsVectorFileWriter.writeAsVectorFormat(input_file, rf"C:\Users\Yusuf_08039508010\Desktop\Working_Files\Fiverr\2021\05-May\Division_Region_Area Map\SHP\US ZipCode\{out_filename}.shp", "UTF-8", input_file.crs(), "ESRI Shapefile")
    else:
        print(f, 'is not a valid input file')
        
print('Done Processing..., ', f)

Thank you for reading.

Search nearby places - Comparing three API (Google Places API, Geoapify API and HERE API)

 In the post, I will compare API from three different providers to search nearby places the three API to compare are: Google Places API, Geoapify API and HERE API.

For each of the platforms, you need to register and get a developer API key to use. All the platform offer a limited free API quota to start with.

Google Places API

import requests
import pandas as pd
from datetime import datetime

df = pd.read_csv('datafile.csv')


YOUR_API_KEY = 'AIza......'

i = 1
for row, col in df.iterrows():
    lat = col['Latitude']
    long = col['Longitude']
    print(i, 'Processing...', lat, long)
    
    url = f'https://maps.googleapis.com/maps/api/place/nearbysearch/json?location={lat}%2C{long}&radius=4850&type=laundry&keyword=laundromats&key={YOUR_API_KEY}'

    payload={}
    headers = {}

    response = requests.request("GET", url, headers=headers, data=payload)

    # Get current time...
    now = datetime.now()
    current_time = now.strftime("%Y%m%d__%H%M%S")

    # Write to file....
    with open(fr'JSON folder\\GoogleAPI\\{state_folder}\\{current_time}.json', 'w') as outfile:
        json.dump(response.json(), outfile)

    i = i+1
    
    
print('Done...')

Geoapify API

GeoApify_API_KEY = '378122b08....'

url = 'https://api.geoapify.com/v2/places'

params = dict(
    categories: 'commercial',
    filter: 'rect:7.735282,48.586797,7.756289,48.574457',
    limit: 2000,
    apiKey=f'{GeoApify_API_KEY}'
)

resp = requests.get(url=url, params=params)
data = resp.json()

print(data)

HERE API

HERE_API_KEY = 'WEYn....'
coord = '27.95034271398129,-82.45670935632066' # lat, long
url = f'https://places.ls.hereapi.com/places/v1/discover/here?apiKey={HERE_API_KEY}&at={coord}&laundry'

response = requests.get(url).json()
# print(response.text)

# Get current time...
now = datetime.now()
current_time = now.strftime("%Y%m%d__%H%M%S")


# Write to file....
with open(fr'JSON folder\\{current_time}.json', 'w') as outfile:
    json.dump(response, outfile)
    
print('Done...')

Making an Unofficial Atlas Map of Nigeria General Election

Unofficial Atlas Map of Nigeria Presidential Election Results

Step-by-step guide

Atlas maps

Convert Coordinates in United Nations Code for Trade and Transport Locations (UN/LOCODE) Code List to GIS friendly format

 The table code list at 'UN/LOCODE Code List by Country and Territory' has a column named coordinate. This column contains the geographical coordinates (latitude/longitude) in a format that is not suitable for use in GIS software. The reason is explained on this page by UN.

So, basically it say in order to avoid unnecessary use of non-standard characters and space, the following standard presentation is used: 0000lat 00000long

(lat - Latitude: N or S ; long – Longitude: W or E, only one digit, capital letter)

Where the last two rightmost digits refer to minutes and the first two or three digits refer to the degrees for latitude and longitude respectively. In addition, you must specify N or S for latitude and W or E for longitude, as appropriate.

While this may be a good format for them, it is not a good format for most GIS platforms. Hence there is need to convert it into what the GIS can easily utilize.

This means we will convert coordinate that looks like this '0507N 00722E' to decimal degrees or degree minute and seconds.

unlocode_coord = '0507S 00722E'

unlat, unlong = unlocode_coord.split(' ')


# Handling Latitide...
# --------------------------
# Remove the last characted which will always be either: N or S...
unlat = unlat.replace('N', '').replace('S', '') # .rstrip('N').rstrip('S')
lat_deg = unlat[:2]
lat_min = unlat[-2:]

# Result in DMS... Degrees Minutes
print(f"The result in Degree Munite is: {lat_deg}°{lat_min}'")


# Result in DD.... Decimal Degrees - Since 1° = 60' and 1' = 60"
lat_min_dd = round(float(lat_min)/60, 2)
# Get the fractional and integer parts, we can use: modulo (%) operator or math.modf
lat_sec_dd = int(lat_min_dd) + round(lat_min_dd % 1, 2)/60
# Add the D+M+S...
lat_dd = round(float(lat_deg) + lat_min_dd + lat_sec_dd, 3)
print(f"The result in Decimal Degree is: {lat_dd}°")




# Handling Longitude...
# --------------------------
# Remove the last characted which will always be either: N or S...
unlong = unlong.rstrip('E').rstrip('W')
long_deg = unlong[:3]
long_min = unlong[-2:]

print()
# Result in DMS... Degrees Minutes Seconds
print(f"The result in Degree Munite is: {long_deg}°{long_min}'")

# Result in DD.... Decimal Degrees - Since 1° = 60' and 1' = 60"
long_min_dd = round(float(long_min)/60, 2)
# Get the fractional and integer parts, we can use: modulo (%) operator or math.modf
long_sec_dd = int(long_min_dd) + round(long_min_dd % 1, 2)/60
# Add the D+M+S...
long_dd = round(float(long_deg) + long_min_dd + long_sec_dd, 3)
print(f"The result in Decimal Degree is: {long_dd}°")

You may use the tool on this website to learn more as seen below.

The reverse - from decimal degrees to UN/LOCODE coordinates

# ------------------- For Latitude ---------------------------------

lat = 13.893937

if lat >= 0: # Northern Hermisphere
    # Degree...
    lat_degree = int(lat)
    # Minute
    lat_minute = int((lat - lat_degree) * 60)
    
    lat_result = str(lat_degree) + str(lat_minute) + 'N'
    print(f'Latitude in UN/LOCODE is: {lat_result}')
    
else: # Southern Hermisphere
    lat = abs(lat)

    # Degree...
    lat_degree = int(lat)
    # Minute
    lat_minute = int((lat - lat_degree) * 60)
    
    lat_result = str(lat_degree) + str(lat_minute) + 'S'
    print(f'Latitude in UN/LOCODE is: {lat_result}')
    


# ------------------- For Longitude ---------------------------------

long = -123.893937

if long >= 0: # Northern Hermisphere
    # Degree...
    long_degree = int(long)
    long_degree1 = str(int(long))

    
    if len(long_degree1) == 1:
        long_degree1 = '00' + long_degree1
    elif len(long_degree1) == 2:
        long_degree1 = '0' + long_degree1
    elif len(long_degree1) == 3:
        long_degree1 = long_degree1    
        
    # Minute
    long_minute = int((long - long_degree) * 60)
    
    long_result = str(long_degree1) + str(long_minute) + 'E'
    print(f'Longitude in UN/LOCODE is: {long_result}')
    
else: # Southern Hermisphere
    long = abs(long)

    # Degree...
    long_degree = int(long)
    # Minute
    long_minute = int((long - long_degree) * 60)
    
    long_result = str(long_degree) + str(long_minute) + 'W'
    print(f'Longitude in UN/LOCODE is: {long_result}')

That is it!

Mathematics of successful life in Python

There is this text that trends over the social media the twenty six alphabets are assigned number from one to twenty six and it was used to calculate the percentage of some word as quoted below;-

I found this to be very interesting and meaningful message to share:-
IF:
A = 1 
B = 2 
C = 3  
D = 4
E = 5  
F = 6
G = 7  
H = 8
I = 9  
J = 10  
K = 11  
L = 12
M = 13  
N = 14 
O = 15  
P = 16
Q = 17
R = 18 
S = 19
T = 20
U = 21
V = 22 
W = 23  
X = 24
Y = 25 
Z = 26

THEN,
H+A+R+D+W+O+R+K
8+1+18+4+23+15+18+11 = 98%

K+N+O+W+L+E+D+G+E
11+14+15+23+12+5+4+7+5 = 96%

L+O+V+E
12+15+22+5 = 54%

L+U+C+K
12+21+3+11 = 47%

None of them makes 100%.
Then what makes 100%?
Is it Money? NO!

M+O+N+E+Y
13+15+14+5+25 = 72%

E+D+U+C+A+T+I+O+N
5+4+21+3+1+20+9+15+14 = 92%

Leadership? NO!

L+E+A+D+E+R+S+H+I+P
12+5+1+4+5+18+19+8+9+16 = 97%

Every problem has a solution, only if we perhaps change our ATTITUDE...
A+T+T+I+T+U+D+E = 1+20+20+9+20+21+4+5  = 100%
It is therefore OUR ATTITUDE towards Life and Work that makes OUR Life 100% Successful.

Amazing mathematics
Let's change our Attitude of doing things in life.
Because it's our attitude that is our problem
Not the Devil.
Tusaai Piadin Gideon copied

Let see how we can transform this into a python script.

alphabets = {'A' : 1, 'B' : 2, 'C' : 3, 'D' : 4, 'E' : 5, 'F' : 6, 'G' : 7, 'H' : 8, 'I' : 9, 'J' : 10, 'K' : 11, 'L' : 12, 'M' : 13, 'N' : 14, 'O' : 15, 'P' : 16, 'Q' : 17, 'R' : 18, 'S' : 19, 'T' : 20, 'U' : 21, 'V' : 22, 'W' : 23, 'X' : 24, 'Y' : 25, 'Z' : 26}

solve = 'M+O+N+E+Y'
solve1 = solve.split('+')
solve2 = [ alphabets[a] for a in solve1 ]
solve3 = str(sum(solve2)) + '%'

print(solve3)

That is it!

Working with Dropbox API in python

 In this post, I will explore Dropbox API. If you don't know what Dropbox is, according to its Wikipedia page: "Dropbox is a file hosting service operated by the American company Dropbox, Inc., headquartered in San Francisco, California, U.S. that offers cloud storage, file synchronization, personal cloud, and client software".

Now that you know what dropbox is, lets see how we can do some basic operations such as uploading, renaming, create, copy, move, delete, download, list to files and folders using python API.

First create an app to generate an API TOKEN on the app developer console page after setting up the right permissions for you use case.

There are two common ways/methods of connecting to dropbox via python:-

The first is using a third party module named "dropbox" which you can install using pip install dropbox. The documentation is found on Dropbox for Python web page.

The second way is using requests module based on their official Dropbox API Explorer.

Which ever method you decide to adopt, it is just a matter of preference. In my case I usually use the combination of the two depending on what is easier for what I want to implement.

Both have excellent documentations. In fact, the Dropbox API Explorer is fantastic for visually constructing the end calls you wanted. For example, I want to get list of contents in a folder, I will on lis_folder tab on the left and configure the options as seen. Then click on show code to grab the code for use in python environment.

Just copy the resulting code into your python environment to execute the API.

import requests
import json

url = "https://api.dropboxapi.com/2/files/list_folder"

headers = {
    "Authorization": "Bearer <access-token>",
    "Content-Type": "application/json"
}

data = {
    "path": "/Images"
}

r = requests.post(url, headers=headers, data=json.dumps(data))

The cat API

 The cat API is a public service API all about Cats. This means it is a service where people who like cats share picture and other details about cats and developers uses it in there applications.

The python code below uses the free cat api to search for random cats, fetch there details and organize it into a dataframe table and the download the pictures onto the local disc.

import json
import shutil
import requests

url = 'https://api.thecatapi.com/v1/images/search'

data_list = []
for i in range(101):
    print('Getting random cat image...', i)
    
    # Send requests...
    response = requests.get(url)
    
    # Get values from response...
    data = list(response.json()[0].values())
    
    # Append data to list...
    data_list.append(data)


# Get columns names from response...
cols = list(response.json()[0].keys())

# Create df...
data_list_df = pd.DataFrame(data_list, columns=cols)
# ===========================================



# Download images....

i = 1
for url in data_list_df['url']:
    # Get image extension...
    img_ext = url.split('.')[-1]
    imgfile_name = f'cat_{i}.{img_ext}'
    print('Processsing...', imgfile_name)
    
    # Send requests...
    res = requests.get(url, stream=True)

    # Write image to disc...
    with open(f'cat_image/{imgfile_name}','wb') as f:
        shutil.copyfileobj(res.raw, f)
        
    # Alternative: write image to disc...
    # with open(f'cat_image/{imgfile_name}','wb') as f:
    #     f.write(res.content)
    
    i = i+1
    # break

Similarly, the code above can be adapted for the Dog API or Dog CEO API.

Thank you for following.

Spatial Distribution of Federal Polytechnics in Nigeria

 On this post, I will map the National Board for Technical Education (NBTE) approved Federal Polytechnics in Nigeria to have a sense of on how they are spatially distributed in the country.

There are 40 polytechnics listed on the NBTE web page above, copy the list into a spreadsheet and geocode the addresses. With this geocoded result, we can prepare the spatial distribution of the federal polytechnic schools as seen below:-

QGIS software was then use to prepare the map.

From the spatial distribution, it could be seen that boundaries between Kebbi and Niger states is a potential location a new proposed federal polytechnic. Similarly, we can recommend/proposed new federal polytechnic between Edo - Ondo states and Adamawa - Taraba states as seen in red color below.

Thank you for reading.

Batch Geocoding and Reverse Geocoding using HERE API

A python script for batch geocoding and reverse geocoding using HERE API

HERE API service allows you to submit batch geocoding and reverse geocoding requests.  Your submission must conform with the Input Data guidelines. The Batch Geocoder API handles the geocoding and reverse geocoding asynchronously.

As stated on the API doc page linked above, to retrieve the output of a successful batch request, you must follow the steps below:-

1. Upload your data with a POST request to the resource jobs.
2. Using the RequestId value contained in the response to your data upload request, check the status of the job with a GET request. You can only download the results when the job status is completed
3. Using the RequestId value contained in the response to your data upload request, download the results by sending a GET request.

Batch Geocoding Python Script

Input Data:

Python Script:

import requests
import json
import time
import zipfile
import io
from bs4 import BeautifulSoup
import glob

import pandas as pd


mykey = 'xxxxxxxxxxxxxx' ## Register a HERE MAPS DEVELOPER API

class Batch:
    SERVICE_URL = "https://batch.geocoder.ls.hereapi.com/6.2/jobs"
    jobId = None
    
    
    def __init__(self, apikey=""): ## use a HERE MAPS DEVELOPER API
        self.apikey = apikey
        
            
    def start(self, filename='testfile.csv', indelim=",", outdelim=","):
        
        file = open(filename, 'rb')

        params = {
            "action": "run",
            "apiKey": self.apikey,
            "politicalview":"RUS",
            "gen": 9,
            "maxresults": "1",
            "header": "true",
            "indelim": indelim,
            "outdelim": outdelim,
            "outcols": "displayLatitude,displayLongitude,locationLabel,houseNumber,street,district,city,postalCode,county,state,country",
            "outputcombined": "true",
        }

        response = requests.post(self.SERVICE_URL, params=params, data=file)
        self.__stats (response)
        file.close()
    

    def status (self, jobId = None):

        if jobId is not None:
            self.jobId = jobId
        
        statusUrl = self.SERVICE_URL + "/" + self.jobId
        
        params = {
            "action": "status",
            "apiKey": self.apikey,
        }
        
        response = requests.get(statusUrl, params=params)
        self.__stats (response)
        

    def result (self, jobId = None):

        if jobId is not None:
            self.jobId = jobId
        
        print("Requesting result data ...")
        
        resultUrl = self.SERVICE_URL + "/" + self.jobId + "/result"
        
        params = {
            "apiKey": self.apikey
        }
        
        response = requests.get(resultUrl, params=params, stream=True)
        
        if (response.ok):    
            zipResult = zipfile.ZipFile(io.BytesIO(response.content))
            zipResult.extractall()
            print("File saved successfully")
        
        else:
            print("Error")
            print(response.text)
    

    
    def __stats (self, response):
        if (response.ok):
            parsedXMLResponse = BeautifulSoup(response.text, "lxml")

            self.jobId = parsedXMLResponse.find('requestid').get_text()
            
            for stat in parsedXMLResponse.find('response').findChildren():
                if(len(stat.findChildren()) == 0):
                    print("{name}: {data}".format(name=stat.name, data=stat.get_text()))
            
            # Contruct the zipfile url...
            self.zip_result = f'https://batch.geocoder.ls.hereapi.com/6.2/jobs/{self.jobId}/result?apiKey={mykey}'
            print("Zipfile URL: ", self.zip_result)
            print('-'*30)
            
            # Set delay for the zipfile to be ready for download on HERE server...
            time.sleep(50)
            
            # Download and extract zip file...
            self.r = requests.get(self.zip_result, stream=True)
            self.z = zipfile.ZipFile(io.BytesIO(self.r.content))
            self.z.extractall(r"HERE\\Output files")

        else:
            print(response.text)

if __name__=="__main__":
    
    service = Batch(apikey=mykey) ## use a HERE MAPS DEVELOPER API
    
    spreadsheet_folder = glob.glob(r'HERE\\Input files\\csv_file\\df_newPU\\*.csv')
    for cvs_file in spreadsheet_folder:
        print('Geocoding file...', cvs_file)
        service.start (cvs_file, indelim = ",", outdelim = ",")

Batch Reverse Geocoding Python Script

Input Data:

Python Script:

l# Reverse Geocode....
class Batch:
    SERVICE_URL = "https://batch.geocoder.ls.hereapi.com/6.2/jobs"
    jobId = None
    
    
    def __init__(self, apikey=""): ## use a HERE MAPS DEVELOPER API
        self.apikey = apikey
        
            
    def start(self, filename='togeocode619.csv', indelim=",", outdelim=","):
        
        file = open(filename, 'rb')

        params = {
            "action": "run",
            "apiKey": self.apikey,
            "politicalview":"RUS",
            "gen": 9,
            "maxresults": "1",
            "header": "true",
            "indelim": indelim,
            "outdelim": outdelim,
            "outCols": "recId,latitude,longitude,locationLabel",
            "outputcombined": "true",
            "mode":"retrieveAddresses",
        }

        
        response = requests.post(self.SERVICE_URL, params=params, data=file)
        self.__stats (response)
        file.close()
    

    def status (self, jobId = None):

        if jobId is not None:
            self.jobId = jobId
        
        statusUrl = self.SERVICE_URL + "/" + self.jobId
        
        params = {
            "action": "status",
            "apiKey": self.apikey,
        }
        
        response = requests.get(statusUrl, params=params)
        self.__stats (response)
        

    def result (self, jobId = None):

        if jobId is not None:
            self.jobId = jobId
        
        print("Requesting result data ...")
        
        resultUrl = self.SERVICE_URL + "/" + self.jobId + "/result"
        
        params = {
            "apiKey": self.apikey
        }
        
        response = requests.get(resultUrl, params=params, stream=True)
        
        if (response.ok):    
            zipResult = zipfile.ZipFile(io.BytesIO(response.content))
            zipResult.extractall()
            print("File saved successfully")
        
        else:
            print("Error")
            print(response.text)
    

    
    def __stats (self, response):
        if (response.ok):
            parsedXMLResponse = BeautifulSoup(response.text, "lxml")

            self.jobId = parsedXMLResponse.find('requestid').get_text()
            
            for stat in parsedXMLResponse.find('response').findChildren():
                if(len(stat.findChildren()) == 0):
                    print("{name}: {data}".format(name=stat.name, data=stat.get_text()))
            
            # Contruct the zipfile url...
            self.zip_result = f'https://batch.geocoder.ls.hereapi.com/6.2/jobs/{self.jobId}/result?apiKey={mykey}'
            print("Zipfile URL: ", self.zip_result)
            print('-'*30)
            
            # Set delay for the zipfile to be ready for download on HERE server...
            time.sleep(50)
            
            # Download and extract zip file...
            self.r = requests.get(self.zip_result, stream=True)
            self.z = zipfile.ZipFile(io.BytesIO(self.r.content))
            self.z.extractall(r"HERE\\Output files")

        else:
            print(response.text)

if __name__=="__main__":
    
    service = Batch(apikey=mykey) ## use a HERE MAPS DEVELOPER API
    cvs_file = r"C:\Users\Yusuf_08039508010\Documents\Jupyter_Notebook\2022\Geocode Addresses\19k LatLong in Fr\reverse_geocode_input.csv"
    
    print('Geocoding file...', cvs_file)
    service.start (cvs_file, indelim = ",", outdelim = ",")
    
#     spreadsheet_folder = glob.glob(r'HERE\\Input files\\csv_file\\df_newPU\\*.csv')
#     for cvs_file in spreadsheet_folder:
#         print('Geocoding file...', cvs_file)
#         service.start (cvs_file, indelim = ",", outdelim = ",")

Happy geocoding!