As of January 2020 GLTG has 32,122,836 datapoints. 

 Please don't fetch all of them at once.


Step 1: Create an Account 

Step 2: Acquire Data from API

You can acquire the data from API by using curl command or python library (we provide the example in jupyter notebook in attached)

Using CURL

Get all Sensors in JSON format

Currently, pulling sensors does not require authentication.

InputsOutput typeOutput Example
url

JSON


{ "sensors":[

         { "id":1445,
         "name":"03254520",
         "created":"2018-03-23T15:48:32Z",
         "geoType":"Feature",
         "geometry":{ "type":"Point",
                                 "coordinates":[ -84.44799549,38.9203417,0]
         },
         "properties":{ "name":"03254520",
                                 "huc":{ "huc8":{ "code":"05100101"},
                                              "huc2":{"code":"05" },
                                              "huc4":{"code":"0510"},
                                              "huc6":{"code":"051001"},
                                               "huc_name":"Licking"
                                            },
           "region":"0510",
           "location":"LICKING RIVER AT HWY 536 NEAR ALEXANDRIA, KY",
           "type":{ 
               "title":"United States Geological Survey",
               "network":"NWIS",
               "id":"usgs"
            },
            "popupContent":"03254520",
            "online_status":"online",
            "id":1445
         },
         "min_start_time":"2007-10-01T06:00:00Z",
         "max_end_time":"2020-02-05T12:30:00Z",
         "parameters":[ 
            "discharge-ft3s",
            "discharge-ft3s-qc",
            "dissolved-oxygen-mgl",
            "dissolved-oxygen-mgl-qc",
            "nitrate-nitrite-as-n-mgl",
            "nitrate-nitrite-as-n-mgl-qc",
            "pH",
            "pH-qc",
            "specific-conductance-uScm",
            "specific-conductance-uScm-qc",
            "turbidity-fnu",
            "turbidity-fnu-qc",
            "water-temperature-c",
            "water-temperature-c-qc"

            ],

            ....

    ]

}}



curl -X GET --compressed https://greatlakestogulf.org/geostreams/api/sensors


Authenticate

InputsOutputDetails
  • url
  • email
  • password
X-Auth-TokenUse the token for fetching datapoints


curl -X POST -H 'Content-Type: application/json' -d '{"password": "****", "identifier": "email"}' --compressed -i https://greatlakestogulf.org/geostreams/api/authenticate


Get all Datapoints for Single Sensor

We request that a user not try to pull all datapoints concurrently.  It is preferred that datapoints be pulled in series by sensor id.

InputsOutput TypeDetailsExample Return
  • token
  • sensor_id
  • since
JSONUse X-Auth-Token from authentication


[    

    { "id":96556536,
      "created":"2019-09-27T20:45:42Z",
      "start_time":"2018-06-25T00:00:00Z",
      "end_time":"2018-06-25T00:00:00Z",
      "properties":{ 
         "nitrate-nitrite-inorganic-total-as-n-mgl":"4.16"

       },
      "type":"Feature",
      "geometry":{          "type":"Point",
         "coordinates":[ -90.645,42.5408333,0 ]
       },
      "stream_id":"28",
      "sensor_id":"22",
      "sensor_name":"IL_EPA_WQX-M-13"
   },
  ...
]



curl -X GET -H 'Content-Encoding: application/json' -H 'x-auth-token:token' --compressed 'https://greatlakestogulf.org/geostreams/api/datapoints?sensor_id=22&since=2018-06-01'


Using Python Requests

Get all sensors and save as csv

import requests
import json
import csv
from csv import DictWriter

api_server = r"https://greatlakestogulf.org/geostreams"
output_directory = r"downloads"

sensors = requests.get(api_server + "/api/sensors").json()["sensors"]

with open(output_directory + '/gltg_sensors.csv', 'w') as f:
    fieldnames = [
        'source','name','location', 'longitude', 'latitude', 'max_end_time', 'min_start_time',
        'parameters' , 'huc8', 'huc_name', 'online_status'
    ]

    writer = DictWriter(f, fieldnames=fieldnames)
    writer.writeheader()

    n_sensors = 0
    n_sensors_pos = 0
    for sensor in sensors:
        n_sensors += 1
        parameters_list = []
        for param in sensor['parameters']:
            if param in ['owner','source','unit_code']:
                continue
            if param[-3:] != "-qc":
                parameters_list.append(param + ',\n')
        parameters = "".join(parameters_list)

        huc8 = None
        if 'code' in sensor['properties']['huc']['huc8']:
            huc8 = sensor['properties']['huc']['huc8']['code']
        else:
            huc8 = sensor['properties']['huc']['huc8']

        if len(parameters) == 0:
            n_sensors_pos += 1
            continue

        writer.writerow({
            "source": sensor['properties']['type']['title'],
            "name": sensor['name'],
            "location": sensor['properties'].get('location', ""),
            'longitude': str(sensor['geometry']['coordinates'][0]),
            'latitude': str(sensor['geometry']['coordinates'][1]),
            'max_end_time': sensor.get('max_end_time',''),
            'min_start_time': sensor.get('min_start_time',''),
            'parameters': parameters,
            'huc8': huc8,
            'huc_name': sensor['properties']['huc'].get('huc_name',''),
            'online_status': sensor['properties'].get('online_status',"")
        })
        
print("Sensors skipped " + str(n_sensors_pos) + " of Sensors total " + str(len(sensors)))

Get Datapoints by Sensor ID 

We request that a user not try to pull all datapoints concurrently.  It is preferred that datapoints be pulled in series by sensor id.

import requests
import json

sensor_id = 22
api_server = r"https://greatlakestogulf.org/geostreams"
output_directory = r"downloads"
user = {'identifier': '***email***', 'password': '***password***'}

r = requests.post(api_server + '/api/authenticate', data=json.dumps(user), headers={'Content-Type': 'application/json'})
print("Authentication status:", r.status_code, "for", api_server)
headers = {"x-auth-token": r.headers["x-auth-token"], "Content-Encoding": "application/json"}

route = api_server + "/api/datapoints?sensor_id=" + str(sensor_id)

r = requests.get(route, headers=headers)

with open(output_directory + '/datapoints_sensor_' + str(sensor_id) + '.json', 'w') as f:
    f.write(json.dumps(r.json(), indent=2))
    
print("Route: " + route)
print("Request Status:", str(r.status_code))
print("Number of datapoints:", len(r.json()))
print("Datapoint JSON saved to " + output_directory + '/datapoints_sensor_' + str(sensor_id) + '.json')



import requests
import json

sensor_id = 22
api_server = r"https://greatlakestogulf.org/geostreams"
output_directory = r"downloads"
user = {'identifier': '***email***', 'password': '***password***'}

r = requests.post(api_server + '/api/authenticate', data=json.dumps(user), headers={'Content-Type': 'application/json'})
print("Authentication status:", r.status_code, "for", api_server)
headers = {"x-auth-token": r.headers["x-auth-token"], "Content-Encoding": "application/json"}

route = api_server + "/api/datapoints?sensor_id=" + str(sensor_id) + "&format=csv"

r = requests.get(route, headers=headers)

with open(output_directory + '/datapoints_sensor_' + str(sensor_id) + '.csv', 'w') as f:
    f.write(r.text)

print("Route: " + route)
print("Request Status:", str(r.status_code))
print("Datapoint JSON saved to " + output_directory + '/datapoints_sensor_' + str(sensor_id) + '.csv')

Jupyter Notebook 

Jupyter notebook example can be download here geostreams_jupyter.ipynb