How to consume the mobility hindrance LDES?

In order to consume an LDES, there is an LDES client needed. We will explain how the hindrance objects can be consumed as JSON objects with Javascript.

We created a demo script that shows how to parse the Gipod ID, description, time schedule, time period, status, WKT geometry and consequences.

First, create a ‘test.js’ file with following content:

const fs = require('fs');
const RRule = require('rrule').RRule;
const RRuleset = require('rrule').RRuleSet;
const pxd = require('parse-xsd-duration');

const context = JSON.parse(fs.readFileSync('./context.jsonld').toString());
const newEngine = require('@treecg/actor-init-ldes-client').newEngine;

let eventstreamSync; // LDES client
const FOLDER_OF_STATE = `./.ldes`;
const LOCATION_OF_STATE = `./.ldes/state.json`;

// When the consequences of a hindrance zone matches one of below list, the hindrance is returned
// Otherwise the hindrance is filtered
let interestedConsequences = [
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/e4ea1344-aa27-40e8-b5af-94ec5f7956f8",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/8eda1611-902b-4c9a-8b3c-4c23a49d7c5d",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/3c9d3c6e-c5bf-477b-a102-d12654ce5ef0",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/6a71c816-511f-490c-9248-c68ded67ecd9",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/c53813ab-814f-4ff4-8a87-6934c72e175f",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/4981fd46-9536-415b-bd30-e53c0cedd799",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/6dd14722-79ad-4d25-aa0e-91e7fc53877b",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/23c9463a-c199-4db3-a8ce-40a088066cb3",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/cd1bbb8c-503f-4968-8483-01a4c2092d51",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/5f8ff25a-87c7-47c8-9332-35d4fabf4b07",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/7cbd0430-f6d4-4c74-8b8e-21b677e6b3d7",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/10c5101d-31fb-4909-a022-d76f868f7f50",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/ee31fd67-b75e-4499-9ad4-0a595717a9c7",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/5e5a1a0b-eaab-4b98-a5c8-6a4664cdb909",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/e52587c7-5566-4c1c-889c-7fcc947e4c4b",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/b922f580-68d3-471d-8712-8175417ad769",
    "api/v1/taxonomies/mobility-hindrance/consequencetypes/427bc6a6-619c-482d-934c-bda986df18d1"
];

// If run takes longer than 50 minutes, pause the LDES Client
const TIMEOUT = 3000000; // milliseconds
// After pausing, the state will be written to LOCATION_OF_STATE
// When restarting, the state will be read from LOCATION_OF_STATE
setTimeout(() => {
    console.log("Timeout - Pausing the LDES client to save state.");
    eventstreamSync.pause();
}, TIMEOUT);

// Pause when exiting with CTRL+C
// process.on('SIGTERM', function() {
//    console.log("Caught interrupt signal. Pausing the LDES client to save state.");
//    eventstreamSync.pause();
// });

start();

async function start() {
  await harvest();
}

function harvest() {
    return new Promise((resolve, reject) => {
        try {
            let url = "https://private-api.gipod.beta-vlaanderen.be/api/v1/ldes/mobility-hindrances";
            let options = {
                "representation": "Object", //Object or Quads
                "emitMemberOnce": true,
                "disableSynchronization": true,
                "jsonLdContext": context,
                "processedURIsCount": 15000
            };
            let LDESClient = new newEngine();
            // Retrieve state
            const state = loadState();
            if (state === null) {
                eventstreamSync = LDESClient.createReadStream(url, options);
            } else {
                eventstreamSync = LDESClient.createReadStream(url, options, state);
            }

            eventstreamSync.on('data', (member) => {
                const object = member.object;
                console.log("test: " + member.id)
                if (hindranceHasInterestingConsequence(member.object)) {
                    if (object.gipodId && object.gipodId.value) {
                        const gipodId = object.gipodId.value;
                        console.log("Gipod ID: " + gipodId);
                    }

                    if (object.createdOn) {
                        // Use this timestamp to update your database with the latest version (version materialisation)
                        console.log("Created on: " + object.createdOn);
                    }

                    const description = object.description;
                    console.log("Description: " + description);

                    let hindranceStillActive = false;
                    if (object.timeSchedule) {
                        for (let schedule of object.timeSchedule) {
                            console.log("Schedule: " + JSON.stringify(schedule));
                            const periodsFromSchedule = processSchedule(schedule);
                            for (let p of periodsFromSchedule) {
                                const start = p.start;
                                const end = p.end;
                                console.log("Time period: " + start + " - " + end);
                                if (new Date().getTime() < new Date(end).getTime()) hindranceStillActive = true;
                            }
                        }
                    }

                    if (object.period) {
                        for (let p of object.period) {
                            const start = p.start;
                            const end = p.end;
                            console.log("Time period: " + start + " - " + end);
                            if (new Date().getTime() < new Date(end).getTime()) hindranceStillActive = true;
                        }
                    }

                    if (hindranceStillActive) {
                        console.log("Hindrance is still active now or in the future");
                    }

                    if (object.status && object.status.prefLabel) {
                        console.log("Status: " + object.status.prefLabel);
                    }

                    for (let z of object.zone) {
                        const geometry = z['geometry'].wkt;
                        console.log("WKT geometry: " + geometry);

                        if (z.consequence && Array.isArray(z.consequence)) {
                            for (let con of z.consequence) {
                                if (con.prefLabel) {
                                    console.log("Consequence: " + con.prefLabel);
                                }
                            }
                        } else if (z.consequence) {
                            if (z.consequence.prefLabel) {
                                console.log("Consequence: " + z.consequence.prefLabel);
                            }
                        }
                    }
                }

                resolve();
            });
            eventstreamSync.on('pause', () => {
                console.log("Paused!")
                // Export current state, but only when paused!
                let state = eventstreamSync.exportState();
                saveState(state);
            });
            eventstreamSync.on('end', () => {
                console.log("No more data!");
                // Save state
                let state = eventstreamSync.exportState();
                saveState(state);
                resolve();
            });
        } catch (e) {
            console.error(e);
            reject(e);
        }
    });
}

function hindranceHasInterestingConsequence(object) {
    if (object.zone) {
        for (let z of object.zone) {
            if (z.consequence && Array.isArray(z.consequence)) {
                for (let con of z.consequence) {
                    if (interestedConsequences.includes(con.id)) {
                        return true;
                    }
                }
            } else if (z.consequence) {
                if (interestedConsequences.includes(z.consequence.id)) {
                    return true;
                }
            }
        }
    }

    return false;
}

function saveState(clientState) {
    if (!fs.existsSync(FOLDER_OF_STATE)) {
        fs.mkdirSync(FOLDER_OF_STATE, { recursive: true });
    }

    fs.writeFileSync(`${LOCATION_OF_STATE}`, JSON.stringify(clientState));
}

function loadState() {
    if (fs.existsSync(LOCATION_OF_STATE)) {
        return JSON.parse(fs.readFileSync(LOCATION_OF_STATE).toString());
    }
    return null;
}

function processSchedule(schedule) {
    const frequency = schedule.repeatFrequency ? schedule.repeatFrequency : "unknown";
    const repeatCount = schedule.repeatCount ? schedule.repeatCount : "unknown"; // Number of times it occurs.
    const startDate = schedule.startDate ? schedule.startDate : "unknown";
    const endDate = schedule.endDate ? schedule.endDate : "unknown";
    const exceptDateArray = schedule.exceptDate ? schedule.exceptDate : [];
    const byDay = schedule.byDay ? schedule.byDay : []; // Days of the month it takes place.
    const byMonthDay = schedule.byMonthDay ? schedule.byMonthDay : []; // Days of the month it takes place.
    const duration = schedule.duration ? schedule.duration : "unknown"; // How long an event specified by this schedule will last.
    const startTime = schedule.startTime ? schedule.startTime : "unknown";

    let rule = {
        "interval": 1
    };

    //// Add frequency
    switch (frequency) {
        case 'P1W':
            rule["freq"] = RRule.WEEKLY;
            break;
        case 'P1D':
            rule["freq"] = RRule.DAILY;
            break;
        case 'P1Y':
            rule["freq"] = RRule.YEARLY;
            break;
        case 'P1M':
            rule["freq"] = RRule.MONTHLY;
            break;
    }

    //// Add count
    if (repeatCount != "unknown") rule["count"] = repeatCount;

    //// Add weekday
    let byWeekday = [];
    for (let day of byDay) {
        switch (day) {
            case 'http://schema.org/Monday':
                byWeekday.push(RRule.MO);
                break;
            case 'http://schema.org/Tuesday':
                byWeekday.push(RRule.TU);
                break;
            case 'http://schema.org/Wednesday':
                byWeekday.push(RRule.WE);
                break;
            case 'http://schema.org/Thursday':
                byWeekday.push(RRule.TH);
                break;
            case 'http://schema.org/Friday':
                byWeekday.push(RRule.FR);
                break;
            case 'http://schema.org/Saturday':
                byWeekday.push(RRule.SA);
                break;
            case 'http://schema.org/Sunday':
                byWeekday.push(RRule.SU);
                break;
        }
    }
    rule["byweekday"] = byWeekday;

    // Add monthday
    rule["bymonthday"] = byMonthDay;

    //// Add dtstart
    if (startDate != "unknown") rule["dtstart"] = new Date(startDate);

    //// Add until
    if (endDate != "unknown") rule["until"] = new Date(endDate);
    else rule["until"] = new Date(startDate.getTime() + 630720000*1000); // Use 20 years as end

    // Create a ruleset:
    let rruleSet = new RRuleset();
    rruleSet.rrule(new RRule(rule));

    // Add exception dates
    for (let exdate of exceptDateArray) {
        rruleSet.exdate(new Date(exdate));
    }

    // Get all occurrence dates (Date instances):
    let occurences = rruleSet.all();

    // Based on the duration and starting time we can calculate the period intervals
    const durationInSeconds = pxd.default(duration);

    let periods = [];
    for (let occurence of occurences) {
        const year = occurence.getFullYear();
        let month = occurence.getMonth() + 1;
        if (month < 10) month = '0' + month;
        let day = occurence.getDate();
        if (day < 10) day = '0' + day;

        const startOfOccurence = new Date(`${year}-${month}-${day}T${startTime}`);
        const endOfOccurence = new Date(startOfOccurence.getTime() + durationInSeconds*1000);
        periods.push({
            "start": startOfOccurence.toISOString(),
            "end": endOfOccurence.toISOString()
        })
    }
    return periods;
}

Second, create a “package.json” file:

{
  "name": "test",
  "version": "1.0.0",
  "description": "",
  "main": "test.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "@treecg/actor-init-ldes-client": "latest",
    "parse-xsd-duration": "^0.5.0",
    "rdf-data-factory": "^1.1.0",
    "rdf-dereference": "^2.0.0",
    "rdf-store-stream": "^1.3.0",
    "rrule": "^2.7.0"
  }
}

Third, create a “context.jsonld” file:

{
	"@context": [{
		"@base": "https://private-api.gipod.beta-vlaanderen.be",
		"id": "@id",
		"value": "@value",
		"type": "@type",
		"@language": "nl-BE",
		"xsd": "http://www.w3.org/2001/XMLSchema#",
		"dct": "http://purl.org/dc/terms/",
		"dc": "http://purl.org/dc/elements/1.1",
		"PublicDomainOccupancy": "https://data.vlaanderen.be/ns/mobiliteit#Inname",
		"Groundwork": "https://data.vlaanderen.be/ns/mobiliteit#Grondwerk",
		"Work": "https://data.vlaanderen.be/ns/mobiliteit#Werk",
		"Event": "https://data.vlaanderen.be/ns/mobiliteit#Evenement",
		"MobilityHindrance": "https://data.vlaanderen.be/ns/mobiliteit#Mobiliteitshinder",
		"SignalingPermit": "https://data.vlaanderen.be/ns/mobiliteit#Signalisatievergunning",
		"TrenchSynergyRequest": "https://data.vlaanderen.be/ns/mobiliteit#Synergieaanvraag",
		"TrenchSynergy": "https://data.vlaanderen.be/ns/mobiliteit#Synergie",
		"Organisation": "http://www.w3.org/ns/org#Organization",
		"Identifier": "http://www.w3.org/ns/adms#Identifier",
		"ContactOrganisationInRole": "http://www.w3.org/ns/org#Organization",
		"ContactInfoPublic": "https://data.vlaanderen.be/ns/mobiliteit#ContactinfoPubliek",
		"AddressRepresentation": "http://www.w3.org/ns/locn#Address",
		"Geometry": "http://www.w3.org/ns/locn#Geometry",
		"Period": "http://data.europa.eu/m8g/PeriodOfTime",
		"PeriodWithDuration": "https://data.vlaanderen.be/ns/mobiliteit#PeriodeMetDuur",
		"TimeSchedule": "https://schema.org/Schedule",
		"Zone": "https://data.vlaanderen.be/ns/mobiliteit#Zone",
		"identifier": {
			"@id": "http://www.w3.org/ns/adms#identifier",
			"@type": "@id",
			"@container": "@set"
		},
		"Identifier.identifier": {
			"@id": "http://www.w3.org/2004/02/skos/core#notation"
		},
		"assignedByName": {
			"@id": "http://www.w3.org/ns/adms#schemaAgency"
		},
		"address": {
			"@id": "http://www.w3.org/ns/locn#address",
			"@type": "@id"
		},
		"isPublic": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Inname.openbaarDomein"
		},
		"owner": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#beheerder",
			"@type": "@id"
		},
		"description": {
			"@id": "http://purl.org/dc/terms/description",
			"@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString"
		},
		"duration": {
			"@id": "https://data.vlaanderen.be/ns/generiek#Tijdsschema.duur",
			"@type": "xsd:duration"
		},
		"contactname": {
			"@id": "http://xmlns.com/foaf/0.1/name"
		},
		"contactOrganisation": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#contactOrganisatie",
			"@type": "@id",
			"@container": "@set"
		},
		"telephone": {
			"@id": "http://schema.org/telephone",
			"@container": "@set"
		},
		"preferredName": {
			"@id": "http://www.w3.org/2004/02/skos/core#prefLabel",
			"@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString"
		},
		"category": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Grondwerk.categorie",
			"@type": "@id"
		},
		"specification": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#specificatie",
			"@type": "@id"
		},
		"isRelocationGroundwork": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Grondwerk.verplaatsingswerk",
			"@type": "xsd:boolean"
		},
		"causingGroundwork": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#heeftOorzaakGrondwerk",
			"@type": "@id",
			"@container": "@set"
		},
		"period": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#periode",
			"@type": "@id",
			"@container": "@set"
		},
		"start": {
			"@id": "http://data.europa.eu/m8g/startTime",
			"@type": "xsd:dateTime"
		},
		"end": {
			"@id": "http://data.europa.eu/m8g/endTime",
			"@type": "xsd:dateTime"
		},
		"periodDuration": {
			"@id": "http://schema.org/duration",
			"@type": "xsd:duration"
		},
		"timeSchedule": {
			"@id": "http://schema.org/eventSchedule",
			"@type": "@id",
			"@container": "@set"
		},
		"startDate": {
			"@id": "http://schema.org/startDate",
			"@type": "xsd:date"
		},
		"startTime": {
			"@id": "http://schema.org/startTime",
			"@type": "xsd:time"
		},
		"endDate": {
			"@id": "http://schema.org/endDate",
			"@type": "xsd:date"
		},
		"endTime": {
			"@id": "http://schema.org/endTime",
			"@type": "xsd:time"
		},
		"consequence": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#gevolg",
			"@type": "@id"
		},
		"status": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Inname.status",
			"@type": "@id"
		},
		"publicDomainOccupancyType": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Inname.type",
			"@type": "@id",
			"@container": "@set"
		},
		"hasConsequence": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Inname.heeftGevolg",
			"@type": "@id"
		},
		"estimatedDuration": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#geschatteDuur",
			"@type": "xsd:duration"
		},
		"zoneType": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#Zone.type",
			"@type": "@id",
			"@container": "@set"
		},
		"byDay": {
			"@id": "http://schema.org/byDay"
		},
		"byMonth": {
			"@id": "http://schema.org/byMonth",
			"@type": "xsd:integer"
		},
		"byMonthDay": {
			"@id": "http://schema.org/byMonthDay",
			"@type": "xsd:integer",
			"@container": "@set"
		},
		"repeatCount": {
			"@id": "http://schema.org/repeatCount",
			"@type": "xsd:integer"
		},
		"repeatFrequency": {
			"@id": "http://schema.org/repeatFrequency"
		},
		"exceptDate": {
			"@id": "http://schema.org/exceptDate",
			"@type": "xsd:date",
			"@container": "@set"
		},
		"bySetPos": {
			"@id": "https://data.vlaanderen.be/ns/generiek#perSetpositie",
			"@type": "xsd:integer",
			"@container": "@set"
		},
		"geometry": {
			"@id": "http://www.w3.org/ns/locn#geometry",
			"@type": "@id"
		},
		"gipodId": {
			"@id": "https://gipod.vlaanderen.be/ns/gipod#gipodId"
		},
		"isConsequenceOf": {
			"@reverse": "https://data.vlaanderen.be/ns/mobiliteit#Inname.heeftGevolg",
			"@type": "@id"
		},
		"permittedBy": {
			"@id": "https://gipod.vlaanderen.be/ns/gipod#isVergundDoor",
			"@type": "@id"
		},
		"prefLabel": {
			"@id": "http://www.w3.org/2004/02/skos/core#prefLabel"
		},
		"reference": {
			"@id": "https://gipod.vlaanderen.be/ns/gipod#reference"
		},
		"remark": {
			"@id": "https://gipod.vlaanderen.be/ns/gipod#remark",
			"@type": "http://www.w3.org/1999/02/22-rdf-syntax-ns#langString"
		},
		"addressRepresentation": {
			"@id": "https://www.w3.org/ns/locn#address",
			"@type": "@id"
		},
		"wkt": {
			"@id": "http://www.opengis.net/ont/geosparql#asWKT",
			"@type": "http://www.opengis.net/ont/geosparql#wktLiteral"
		},
		"zone": {
			"@id": "https://data.vlaanderen.be/ns/mobiliteit#zone",
			"@type": "@id",
			"@container": "@set"
		},
		"lastModifiedOn": {
			"@id": "dct:modified",
			"@type": "xsd:dateTime"
		},
		"lastModifiedBy": {
			"@id": "http://purl.org/dc/elements/1.1/contributor",
			"@type": "@id"
		},
		"createdOn": {
			"@id": "dct:created",
			"@type": "xsd:dateTime"
		},
		"createdBy": {
			"@id": "http://purl.org/dc/elements/1.1/creator",
			"@type": "@id"
		}
	}, {
		"adms": "http://www.w3.org/ns/adms#",
		"prov": "http://www.w3.org/ns/prov#",
		"ldes": "https://w3id.org/ldes#",
		"tree": "https://w3id.org/tree#",
		"eventName": "adms:versionNotes",
		"EventStream": "ldes:EventStream",
		"Node": "tree:Node",
		"view": "tree:view",
		"viewOf": {
			"@reverse": "view",
			"@type": "@id"
		},
		"member": "tree:member",
		"relation": "tree:relation",
		"memberOf": {
			"@reverse": "member",
			"@type": "@id"
		},
		"timestampPath": {
			"@id": "ldes:timestampPath",
			"@type": "@id"
		},
		"versionOfPath": {
			"@id": "ldes:versionOfPath",
			"@type": "@id"
		},
		"shape": {
			"@id": "tree:shape",
			"@type": "@id"
		},
		"tree:node": {
			"@type": "@id"
		},
		"tree:path": {
			"@type": "@id"
		},
		"tree:value": {
			"@type": "xsd:dateTime"
		},
		"generatedAtTime": {
			"@id": "prov:generatedAtTime",
			"@type": "xsd:dateTime"
		},
		"isVersionOf": {
			"@id": "dct:isVersionOf",
			"@type": "@id"
		},
		"items": "@included",
		"collectionInfo": "@included"
	}]
}

And install the packages with following command in the terminal:

Now we can run the script as follows:

The script currently uses a test LDES with one hindrance object.

The outputted JSON object, which you will process further, looks like this:

The script shows how to parse the WKT geometry, time period, schedule etc.

The function hindranceHasInterestingConsequence demonstrates how objects can be filtered on consequence type.

The result of the script is:

Restarting

In the script test.js, we also pause the stream after 50 minutes. This can be configured with the TIMEOUT property:

const TIMEOUT = 3000000; // milliseconds

setTimeout(() => eventstreamSync.pause(), TIMEOUT);

When pausing, this triggers the function “saveState”, which will save the state to the .ldes/state.json file.

When resuming, this state will be picked up again.

If you want to reset the client, remove the state file.

GIPOD COCREATIE

Example LDES Implementation Guide

How to consume the mobility hindrance LDES?

Restarting