...
python3 ./nbiCsvJsonConverter-2/ProcessMain.py
Converts ingests the data to JSON formatinto a MongoDB
Data is 6.3 GB uncompressed CSV
MongoDBÂ
Getting the data to Workbench
...
Code Block |
---|
{ "_id" : ObjectId("59b8519bf6b8e300bb668a93"), "year" : 1992, "stateCode" : "02", "structureNumber" : "0175", "inventoryRoute" : { "recordType" : "1", "routeSigningPrefix" : -1, "designatedLevelOfService" : -1, "routeNumber" : "NA", "directionalSuffix" : -1 }, "highwayAgencyDistrict" : "00", "countyCode" : 0, "placeCode" : 0, "featuresIntersected" : { "featuresIntersected" : "NA", "criticalFacilityIndicator" : "NA" }, "facilityCarriedByStructure" : "NA", "location" : "NA", "InventoryRTeMinVertClearance" : 0, "kilometerpoint" : -1, "baseHighwayPoint" : -1, "inventoryRouteSubrouteNumber" : { "LRSInventoryRoute" : "NA" }, "latitude" : 0, "longitude" : 0, "bypassDetourLength" : 0, "toll" : -1, "maintenanceReponsibility" : -1, "owner" : -1, "functionalClassOfInventoryRte" : -1, "yearBuilt" : -1, "lanesOnUnderStructure" : { "lanesOnStructure" : -1, "lanesUnderStructure" : 0 }, "averageDailyTraffic" : 0, "yearOfAverageDailyTraffic" : -1, "designLoad" : 0, "approachRoadwayWidth" : 0, "bridgeMedian" : 0, "skew" : 0, "structureFlared" : 0, "trafficSafetyFeatures" : { "bridgeRailings" : "NA", "transitions" : "NA", "approachGuardrail" : "NA", "approachGuardrailEnds" : "NA" }, "historicalSignificance" : -1, "navigationControl" : "NA", "navigationVeriticalClearance" : 0, "navigationHorizontalClearance" : 0, "strucutreOpenPostedClosed" : "NA", "typeOfService" : { "typeOfServiceOnBridge" : 0, "typeOfServiceUnderBridge" : 0 }, "structureTypeMain" : { "kindOfMaterialDesign" : 0, "typeOfDesignConstruction" : 0 }, "structureTypeApproachSpans" : { "kindOMaterialDesign" : 0, "typeOfDesignContruction" : 0 }, "numberOfSpansInMainUnit" : 0, "numberOfApproachSpans" : 0, "InventoryRteTotalHorzClearance" : 0, "lengthOfMaximumSpan" : 0, "structureLength" : 0, "curbSidewalk Width" : { "leftCurbSidewalkWidth" : 0, "rightCurbSidewalkWidth" : 0 }, "bridgeRoadwayWithCurbToCurb" : 0, "deckWidthOutToOut" : 0, "minVertClearOverBridgeRoadway" : 0, "minimumVeriticalUnderclearance" : { "referenceFeature" : "NA", "minimumVeriticalUnderclearance" : -1 }, "minLateralUnderclearOnRight" : { "referenceFeature" : "NA", "minimumLateralUnderclearance" : -1 }, "minLateralUnderclearOnLeft" : -1, "deck" : "NA", "superstructure" : "NA", "substructure" : "NA", "channelChannelProtection" : "NA", "culverts" : "NA", "methodUsedToDetermineOperatingRating" : -1, "operatingRating" : 0, "methodUsedToDetermineInventoryRating" : -1, "inventoryRating" : 0, "structuralEvaluation" : "NA", "deckGeometry" : "NA", "underclearVerticalHorizontal" : "N", "bridgePosting" : -1, "waterwayAdequacy" : "NA", "approachRoadwayAlignment" : "NA", "typeOfWork" : { "typeOfWorkProposed" : -1, "WorkDoneBy" : "NA" }, "lengthOfStructureImprovement" : 0, "inspectionDate" : -1, "designatedInspectionFrequency" : -1, "criticalFeatureInspection" : { "fractureCriticalDetails" : "NA", "underwaterInspection" : "NA", "otherSpecialInspection" : "NA" }, "criticalFeatureInspectionDates" : { "fractureCiritcalDetailsDate" : "NA", "underwaterInspectionDate" : "NA", "OtherSpecialInspectionDate" : "NA" }, "bridgeImprovementCost" : 0, "roadwayImprovementCost" : 0, "totalProjectCost" : 0, "yearOfImprovementCost" : 2000, "borderBridge" : { "neighboringStateCode" : "NA", "percentReponsibility" : -1 }, "borderBridgeStructureNumber" : "NA", "STRAHNETHighwayDesignation" : -1, "parallelStructureDesignation" : "NA", "directionOfTraffic" : -1, "temporaryStructureDesignation" : "NA", "highwaySystemOfInventoryRoute" : -1, "federalLandsHighways" : -1, "yearReconstructed" : 0, "deckStructureType" : "NA", "wearingSurface/ProtectiveSystem" : { "typeOfWearingSurface" : "NA", "typeOfMembrane" : "NA", "deckProtection" : "NA" }, "avgDailyTruckTraffic" : -1, "designatedNationalNetwork" : -1, "pier/abutmentProtection" : -1, "nbisBridgeLength" : "NA", "scourCriticalBridges" : "NA", "futureAvgDailyTraffic" : 0, "yearOfFutureAvgDailyTraffic" : 2000, "minimumNavigationVerticalClearanceVerticalLiftBridge" : 0, "federalAgencyIndicator" : "N", "dateLastUpdate" : "NA", "typeLastUpdate" : "NA", "deductCode" : "Z", "status with 10 year rule" : "N", "sufficiencyRatingAsteriskField" : "NA", "sufficiencyRating" : -1, "loc" : { "type" : "Point", "coordinates" : [ 0, 0 ] } } |
Jupyter Notebook
http://faculty.ist.unomaha.edu/rgandhi/r/mongoNBI.html
Dependencies: pymongo, pandas, gridfs (image data only)
Apache Spark
Mongo is only useful for a certain kind of data
Metadata record
Do we host a record describing this dataset
...