You are viewing an old version of this page. View the current version.
Compare with Current
View Page History
« Previous
Version 55
Next »
As of January 2020 GLTG has 32,122,836 datapoints.
WARNING Please don't fetch all of them at once.
Step 1: Create an Account
Step 2: Acquire Data from API
You can acquire the data from API by using curl command or python library (we provide the example in jupyter notebook in attached)
Using CURL
Currently, pulling sensors does not require authentication.
Inputs | Output type | Output Example |
---|
url | JSON |
{ "sensors":[
{ "id":1445,
"name":"03254520",
"created":"2018-03-23T15:48:32Z",
"geoType":"Feature",
"geometry":{ "type":"Point",
"coordinates":[ -84.44799549,38.9203417,0]
},
"properties":{ "name":"03254520",
"huc":{ "huc8":{ "code":"05100101"},
"huc2":{"code":"05" },
"huc4":{"code":"0510"},
"huc6":{"code":"051001"},
"huc_name":"Licking"
},
"region":"0510",
"location":"LICKING RIVER AT HWY 536 NEAR ALEXANDRIA, KY",
"type":{
"title":"United States Geological Survey",
"network":"NWIS",
"id":"usgs"
},
"popupContent":"03254520",
"online_status":"online",
"id":1445
},
"min_start_time":"2007-10-01T06:00:00Z",
"max_end_time":"2020-02-05T12:30:00Z",
"parameters":[
"discharge-ft3s",
"discharge-ft3s-qc",
"dissolved-oxygen-mgl",
"dissolved-oxygen-mgl-qc",
"nitrate-nitrite-as-n-mgl",
"nitrate-nitrite-as-n-mgl-qc",
"pH",
"pH-qc",
"specific-conductance-uScm",
"specific-conductance-uScm-qc",
"turbidity-fnu",
"turbidity-fnu-qc",
"water-temperature-c",
"water-temperature-c-qc"
],
....
]
}}
|
curl -X GET --compressed https://greatlakestogulf.org/geostreams/api/sensors
Authenticate
Inputs | Output | Details |
---|
| X-Auth-Token | Use the token for fetching datapoints |
curl -X POST -H 'Content-Type: application/json' -d '{"password": "****", "identifier": "email"}' --compressed -i https://greatlakestogulf.org/geostreams/api/authenticate
Get all Datapoints for Single Sensor
We request that a user not try to pull all datapoints concurrently. It is preferred that datapoints be pulled in series by sensor id.
Inputs | Output Type | Details | Example Return |
---|
| JSON | Use X-Auth-Token from authentication |
[
{ "id":96556536,
"created":"2019-09-27T20:45:42Z",
"start_time":"2018-06-25T00:00:00Z",
"end_time":"2018-06-25T00:00:00Z",
"properties":{
"nitrate-nitrite-inorganic-total-as-n-mgl":"4.16"
},
"type":"Feature",
"geometry":{ "type":"Point",
"coordinates":[ -90.645,42.5408333,0 ]
},
"stream_id":"28",
"sensor_id":"22",
"sensor_name":"IL_EPA_WQX-M-13"
},
...
]
|
curl -X GET -H 'Content-Encoding: application/json' -H 'x-auth-token:token' --compressed 'https://greatlakestogulf.org/geostreams/api/datapoints?sensor_id=22&since=2018-06-01'
Using Python Requests
Get all sensors and save as csv
import requests
import json
import csv
from csv import DictWriter
api_server = r"https://greatlakestogulf.org/geostreams"
output_directory = r"downloads"
sensors = requests.get(api_server + "/api/sensors").json()["sensors"]
with open(output_directory + '/gltg_sensors.csv', 'w') as f:
fieldnames = [
'source','name','location', 'longitude', 'latitude', 'max_end_time', 'min_start_time',
'parameters' , 'huc8', 'huc_name', 'online_status'
]
writer = DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
n_sensors = 0
n_sensors_pos = 0
for sensor in sensors:
n_sensors += 1
parameters_list = []
for param in sensor['parameters']:
if param in ['owner','source','unit_code']:
continue
if param[-3:] != "-qc":
parameters_list.append(param + ',\n')
parameters = "".join(parameters_list)
huc8 = None
if 'code' in sensor['properties']['huc']['huc8']:
huc8 = sensor['properties']['huc']['huc8']['code']
else:
huc8 = sensor['properties']['huc']['huc8']
if len(parameters) == 0:
n_sensors_pos += 1
continue
writer.writerow({
"source": sensor['properties']['type']['title'],
"name": sensor['name'],
"location": sensor['properties'].get('location', ""),
'longitude': str(sensor['geometry']['coordinates'][0]),
'latitude': str(sensor['geometry']['coordinates'][1]),
'max_end_time': sensor.get('max_end_time',''),
'min_start_time': sensor.get('min_start_time',''),
'parameters': parameters,
'huc8': huc8,
'huc_name': sensor['properties']['huc'].get('huc_name',''),
'online_status': sensor['properties'].get('online_status',"")
})
print("Sensors skipped " + str(n_sensors_pos) + " of Sensors total " + str(len(sensors)))
Get Datapoints by Sensor ID
We request that a user not try to pull all datapoints concurrently. It is preferred that datapoints be pulled in series by sensor id.
import requests
import json
sensor_id = 22
api_server = r"https://greatlakestogulf.org/geostreams"
output_directory = r"downloads"
user = {'identifier': '***email***', 'password': '***password***'}
r = requests.post(api_server + '/api/authenticate', data=json.dumps(user), headers={'Content-Type': 'application/json'})
print("Authentication status:", r.status_code, "for", api_server)
headers = {"x-auth-token": r.headers["x-auth-token"], "Content-Encoding": "application/json"}
route = api_server + "/api/datapoints?sensor_id=" + str(sensor_id)
r = requests.get(route, headers=headers)
with open(output_directory + '/datapoints_sensor_' + str(sensor_id) + '.json', 'w') as f:
f.write(json.dumps(r.json(), indent=2))
print("Route: " + route)
print("Request Status:", str(r.status_code))
print("Number of datapoints:", len(r.json()))
print("Datapoint JSON saved to " + output_directory + '/datapoints_sensor_' + str(sensor_id) + '.json')
import requests
import json
sensor_id = 22
api_server = r"https://greatlakestogulf.org/geostreams"
output_directory = r"downloads"
user = {'identifier': '***email***', 'password': '***password***'}
r = requests.post(api_server + '/api/authenticate', data=json.dumps(user), headers={'Content-Type': 'application/json'})
print("Authentication status:", r.status_code, "for", api_server)
headers = {"x-auth-token": r.headers["x-auth-token"], "Content-Encoding": "application/json"}
route = api_server + "/api/datapoints?sensor_id=" + str(sensor_id) + "&format=csv"
r = requests.get(route, headers=headers)
with open(output_directory + '/datapoints_sensor_' + str(sensor_id) + '.csv', 'w') as f:
f.write(r.text)
print("Route: " + route)
print("Request Status:", str(r.status_code))
print("Datapoint JSON saved to " + output_directory + '/datapoints_sensor_' + str(sensor_id) + '.csv')
Jupyter notebook example can be download here geostreams_jupyter.ipynb