From 29292f78974d9b9e71bf63ce207c3037f99a33fb Mon Sep 17 00:00:00 2001 From: Markus Opolka <markus@martialblog.de> Date: Sat, 13 Jan 2018 09:25:48 +0100 Subject: [PATCH] Refactor API - Simpler library - Add caching --- TODO.md | 1 - api.js | 177 +++++++++++++++++++++--- api.legacy.js | 62 +++++++++ lib.js | 263 +++++++++++------------------------- lib.legacy.js | 253 ++++++++++++++++++++++++++++++++++ package.json | 3 +- src/components/Features.vue | 2 +- 7 files changed, 560 insertions(+), 201 deletions(-) create mode 100644 api.legacy.js create mode 100644 lib.legacy.js diff --git a/TODO.md b/TODO.md index 849f492..9a14ed5 100644 --- a/TODO.md +++ b/TODO.md @@ -2,4 +2,3 @@ - Besser Bildanzeige - Features in Brief Ansicht - Redesign Brief Component -- require('./a').a für lib.js diff --git a/api.js b/api.js index e38c53a..1058a11 100755 --- a/api.js +++ b/api.js @@ -1,12 +1,127 @@ const express = require('express') const path = require('path') +const fs = require('fs') +const crypto = require('crypto') +const cache = require('memory-cache') +const convert = require('xml-js') + const lib = require('./lib.js') -// Setting -const dir = 'letters' +// Settings +const DIR = 'letters' const app = express() const PORT = 3000 +function loadDocAsJson (filepath) { + /* + * TODO: Maybe check if XML is valid? + * Reads XML file and transforms it to JSON + */ + + let data = {} + + try { + let xml = fs.readFileSync(filepath, 'utf8') + data = convert.xml2js(xml, {compact: true}) + } catch (err) { + console.log('> Error while loading ' + filepath) + console.log(err) + return data + } + + return data +} + +function inCacheFormat (docname, docobj) { + /* + * Create some metadata for document cache representation + */ + + let hash = crypto.createHmac('sha256', JSON.stringify(docobj)).digest('hex') + + return { + link: '/letters/' + docname.slice(0, -4), + collection: docobj.TEI.teiHeader.fileDesc.titleStmt.collection._text, + url: '/api/letters/' + docname.slice(0, -4), + name: docname.replace(/_/g, ' '), + hash: hash, + object: docobj + } +} + +function loadDocumentsToCache (dir) { + /* + * Load all documents to cache + */ + + try { + var documents = fs.readdirSync(dir) + } catch (err) { + console.log('> Error while loading files from ' + dir) + console.log(err) + } + + for (let docname of documents) { + // We just want the XML documents + if (!docname.endsWith('.xml')) { + continue + } + let docjs = loadDocAsJson(path.join(dir, docname)) + cache.put(docname, inCacheFormat(docname, docjs)) + } + + console.log('> Loaded all documents to cache') +} + +function loadDocuments (dir) { + /* + * Load documents from cache and update cache if necessary + */ + + // Update the cache + // TODO: If I load the file each time, I gain nothing + + // try { + // var documents = fs.readdirSync(dir) + // var cachedocments = cache.keys() + // } catch (err) { + // console.log('> Error while loading files from ' + dir) + // } + + // // Hint: gitignore is in documents + // if ((documents.length -1) != cachedocments.length) { + // console.log('> Refreshing documents cache') + // loadDocumentsToCache(dir) + // } + + let data = [] + + for (let item of cache.keys()) { + data.push(cache.get(item)) + } + + return data +} + +function loadDocument (doc) { + /* + * Load document from cache and update cache if necessary + */ + + // Update the cache + // TODO: If I load the file each time, I gain nothing + + // var docjs = loadDocAsJson(path.join(DIR, doc)) + // const hash = crypto.createHmac('sha256', JSON.stringify(docjs)).digest('hex') + + // if (hash != cache.get(doc).hash) { + // console.log('> Refreshing cache for ' + doc) + // cache.put(doc, inCacheFormat(doc, docjs)) + // } + + return cache.get(doc) +} + // CORS Settings app.use(function (req, res, next) { res.header('Access-Control-Allow-Origin', '*') @@ -14,49 +129,77 @@ app.use(function (req, res, next) { next() }) -app.get('/api/search', function(req, res) { +app.get('/api/search', function (req, res) { // Search Endpoint with query parameters // <element attribute="value">content</element> - var query = { + let query = { element: req.query.element, content: req.query.content, attribute: req.query.attribute, value: req.query.value } - var data = lib.forAllFiles(lib.searchLetters, dir, {query:query}) + let data = [] + for (let item of cache.keys()) { + let obj = loadDocument(item) + let res = lib.search(obj, query) + if (res !== null) { + data.push(res) + } + } + res.send(data) }) -app.get('/api/feature/', function(req, res) { +app.get('/api/features/:letter?', function (req, res) { // Feature Search Endpoint with query parameters // <feature category="diakritika" type="akut" subtype="akutstattgravis" ref="26"> - var query = { + + let query = { category: req.query.category, type: req.query.type, subtype: req.query.subtype } - var data = lib.forAllFiles(lib.searchFeatures, dir, {query:query}) - res.send(data) + let data = [] + const DOC = req.params.letter + '.xml' + + // TODO: Refactor this shice + if (req.params.letter) { + let obj = loadDocument(DOC) + let res = lib.features(obj, query) + if (res !== null) { + data.push(res) + } + } else { + for (let item of cache.keys()) { + let obj = loadDocument(item) + let res = lib.features(obj, query) + if (res !== null) { + data.push(res) + } + } + + res.send(data) + } }) -app.get('/api/letters/:filename?', function (req, res) { +app.get('/api/letters/:letter?', function (req, res) { // Document Endpoint with optional Document name // Lists either all or one given document - if (req.params.filename){ - // Load specified document - var filename = req.params.filename + '.xml' - res.send(lib.getXMLasJSON(path.join(dir, filename))) + const DOC = req.params.letter + '.xml' + + if (req.params.letter) { + res.send(loadDocument(DOC).object) } else { - // Load all documents - var data = lib.forAllFiles(lib.listLetters, dir) - res.send(data) + res.send(loadDocuments(DIR)) } }) // Where the magic happens +loadDocumentsToCache(DIR) + app.listen(PORT, function () { console.log('> Listening at http://localhost:' + PORT) }) diff --git a/api.legacy.js b/api.legacy.js new file mode 100644 index 0000000..6c3c99c --- /dev/null +++ b/api.legacy.js @@ -0,0 +1,62 @@ +const express = require('express') +const path = require('path') +const lib = require('./lib.legacy.js') + +// Setting +const dir = 'letters' +const app = express() +const PORT = 3000 + +// CORS Settings +app.use(function (req, res, next) { + res.header('Access-Control-Allow-Origin', '*') + res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept') + next() +}) + +app.get('/api/search', function(req, res) { + // Search Endpoint with query parameters + // <element attribute="value">content</element> + var query = { + element: req.query.element, + content: req.query.content, + attribute: req.query.attribute, + value: req.query.value + } + + var data = lib.forAllFiles(lib.searchLetters, dir, {query:query}) + res.send(data) +}) + +app.get('/api/features/', function(req, res) { + // Feature Search Endpoint with query parameters + // <feature category="diakritika" type="akut" subtype="akutstattgravis" ref="26"> + var query = { + category: req.query.category, + type: req.query.type, + subtype: req.query.subtype + } + + var data = lib.forAllFiles(lib.searchFeatures, dir, {query:query}) + res.send(data) +}) + +app.get('/api/letters/:filename?', function (req, res) { + // Document Endpoint with optional Document name + // Lists either all or one given document + + if (req.params.filename){ + // Load specified document + var filename = req.params.filename + '.xml' + res.send(lib.getXMLasJSON(path.join(dir, filename))) + } else { + // Load all documents + var data = lib.forAllFiles(lib.listLetters, dir) + res.send(data) + } +}) + +// Where the magic happens +app.listen(PORT, function () { + console.log('> Listening at http://localhost:' + PORT) +}) diff --git a/lib.js b/lib.js index 827e7a2..2b6f16a 100755 --- a/lib.js +++ b/lib.js @@ -1,16 +1,14 @@ -const fs = require('fs') -const path = require('path') -const convert = require('xml-js') -const flat = require('flat') -const flatten = require('flatten') -const dir = 'letters' - -function findKeyRecursive(object, keytofind, ret = []) { +const flat = require('flat') // Arrays +const flatten = require('flatten') // Objects + +function findKeyRecursive (object, keytofind, ret = []) { /* * Finds all keys in a nested object and returns them as an array */ + var value - Object.keys(object).forEach(function(key) { + + Object.keys(object).forEach(function (key) { if (key === keytofind) { ret.push(object[key]) return true @@ -18,91 +16,65 @@ function findKeyRecursive(object, keytofind, ret = []) { if (object[key] && typeof object[key] === 'object') { value = findKeyRecursive(object[key], keytofind, ret) return value !== undefined - } - else { + } else { return true } }) return flatten(ret) } -function searchInFeatures(features, query) { +function countReferences (features) { /* - * Searches for a query in an array of features - * Attributes must be in _attributes + * Count the references in a feature */ - var results = [] - var reCat = new RegExp(query.category) - var reType = new RegExp(query.type) - var reSub = new RegExp(query.subtype) - - for(var idx = 0; idx < features.length; idx++){ - var match_category = reCat.test(features[idx]._attributes.category) - var match_type = reType.test(features[idx]._attributes.type) - var match_subtype = reSub.test(features[idx]._attributes.subtype) + let count = 0 - if (match_category && match_type && match_subtype) { - results.push(features[idx]) - } + for (let idx = 0; idx < features.length; idx++) { + count = count + features[idx]._attributes.ref.trim().split(' ').length } - return results -} -function transformFilename (filename, url = false) { - /* - * Transforms a full filepath: - * from example_1.xml to example 1 - * URL: from example_1.xml to example_1 - */ - filename = filename.slice(0, -4) - - if (!url) { - filename = filename.replace(/_/g, ' ') - } - - return filename + return count } function searchInObject (obj, query) { /* * Search for query in a fiven flatobject */ - var results = [] + let results = [] Object.keys(obj).forEach(function (key) { - // Attributes - if (typeof query.attribute !== "undefined"){ - var re_attr = new RegExp('_attributes.' + query.attribute) - if (!re_attr.test(key)){ + if (typeof query.attribute !== 'undefined') { + const reAttr = new RegExp('_attributes.' + query.attribute) + if (!reAttr.test(key)) { return } } // Values - if (typeof query.value !== "undefined"){ - var re_valu = new RegExp(query.value) - if (!re_valu.test(obj[key])){ + if (typeof query.value !== 'undefined') { + const reVal = new RegExp(query.value) + if (!reVal.test(obj[key])) { return } } // Elements - if (typeof query.element !== "undefined"){ - var re_elem = new RegExp('[^_]' + query.element, 'g') - if (!re_elem.test(key)){ + if (typeof query.element !== 'undefined') { + const reElem = new RegExp('[^_]' + query.element, 'g') + if (!reElem.test(key)) { return } } // Content - if (typeof query.content !== "undefined"){ - var re_cont = new RegExp(query.content) - if (!re_cont.test(obj[key])){ + if (typeof query.content !== 'undefined') { + const reCont = new RegExp(query.content) + if (!reCont.test(obj[key])) { return } } // Add hit to results as object - res = {} + let res = {} res[key] = obj[key] results.push(res) }) @@ -110,144 +82,73 @@ function searchInObject (obj, query) { return results } -function getXMLasJSON (filepath) { +function searchInFeatures (features, query) { /* - * TODO: This is duplicate Code. Needs to go - * TODO: Maybe check if XML is valid? - * Reads XML file and transforms it to JSON + * Searches for a query in an array of features + * Attributes must be in _attributes */ - var data = {} - try { - var xml = fs.readFileSync(filepath, 'utf8') - data = convert.xml2js(xml, {compact: true}) - } catch (err) { - console.log('Error while loading ' + filepath) - console.log(err) - return data + + var results = [] + var reCat = new RegExp(query.category) + var reType = new RegExp(query.type) + var reSub = new RegExp(query.subtype) + + for (var idx = 0; idx < features.length; idx++) { + var matchCat = reCat.test(features[idx]._attributes.category) + var matchType = reType.test(features[idx]._attributes.type) + var matchSub = reSub.test(features[idx]._attributes.subtype) + + if (matchCat && matchType && matchSub) { + results.push(features[idx]) + } } - return data + return results } -function countReferences (features) { +function searchFeaturesInDocument (document, query) { /* - * Count the references in a feature + * Search for features in (js) document */ - var count = 0 - for (idx = 0; idx < features.length; idx++) { - count = count + features[idx]._attributes.ref.trim().split(" ").length - } - return count -} + if (typeof query.category === 'undefined') { query.category = '.*' } + if (typeof query.type === 'undefined') { query.type = '.*' } + if (typeof query.subtype === 'undefined') { query.subtype = '.*' } -module.exports = { - getXMLasJSON: function (filepath) { - /* - * Reads XML file and transforms it to JSON - */ - var data = {} - try { - var xml = fs.readFileSync(filepath, 'utf8') - data = convert.xml2js(xml, {compact: true}) - } catch (err) { - console.log('Error while loading ' + filepath) - console.log(err) - return data - } - return data - }, + var features = findKeyRecursive(document, 'feature') + var results = searchInFeatures(features, query) + var count = countReferences(results) - listLetters:function (file, params) { - /* - * Callback function - * Transforms a given parsed XML file into the list view - */ - return { - link: '/letters/' + file.slice(0, -4), - name: transformFilename(file), - filepath: path.join(dir, file), - url: '/api/letters/' + transformFilename(file, true) - } - }, - searchFeatures: function(file, params) { - /* - * Callback function - * Search for features in file - */ - var query = params.query - var result = undefined - var letterJS = getXMLasJSON(path.join(dir, file)) - var features = findKeyRecursive(letterJS, 'feature') - - if (typeof query.category == "undefined"){ query.category = '.*' } - if (typeof query.type == "undefined"){ query.type = '.*' } - if (typeof query.subtype == "undefined"){ query.subtype = '.*' } - - var res = searchInFeatures(features, query) - var count = countReferences(res) - - if(res.length > 0) { - result = { - collection: letterJS.TEI.teiHeader.fileDesc.titleStmt.collection._text, - count: count, - name: transformFilename(file), - link: '/letters/' + file.slice(0, -4), - url: '/api/letters/' + transformFilename(file, true), - results: res - } - } - return result + if (results.length > 0) { + document['count'] = count + document['results'] = results + } else { + return null + } - }, - searchLetters: function(file, params){ - /* - * Callback function - * Search for query in file - */ - var query = params.query - var result = undefined - var letterJS = getXMLasJSON(path.join(dir, file)) - var flatobject = flat(letterJS.TEI) - var res = searchInObject(flatobject, query) - - if(res.length > 0) { - result = { - collection: letterJS.TEI.teiHeader.fileDesc.titleStmt.collection._text, - name: transformFilename(file), - link: '/letters/' + file.slice(0, -4), - url: '/api/letters/' + transformFilename(file, true), - results: res - } - } - return result - }, + return document +} - forAllFiles: function (callback, filepath, params={}) { - /* - * Load files from filepath and perform callback on each with given parameters - */ - var ret = [] - - try { - var files = fs.readdirSync(filepath) - } catch (err) { - console.log('> Error while loading files from ' + filepath) - console.log(err) - return [] - } +function searchInDocument (document, query) { + /* + * Search for query in (js) document + */ + var flatobject = flat(document.object.TEI) + var results = searchInObject(flatobject, query) - for (var i in files) { - // We just want the XML documents - if (!files[i].endsWith('.xml')) { - continue - } + if (results.length > 0) { + document['results'] = results + } else { + return null + } - // Call callback function on each file - var val = callback(files[i], params) - if (typeof val !== "undefined"){ - ret.push(val) - } - } - return ret + return document +} + +module.exports = { + features: function (obj, query) { + return searchFeaturesInDocument(obj, query) + }, + search: function (obj, query) { + return searchInDocument(obj, query) } } diff --git a/lib.legacy.js b/lib.legacy.js new file mode 100644 index 0000000..827e7a2 --- /dev/null +++ b/lib.legacy.js @@ -0,0 +1,253 @@ +const fs = require('fs') +const path = require('path') +const convert = require('xml-js') +const flat = require('flat') +const flatten = require('flatten') +const dir = 'letters' + +function findKeyRecursive(object, keytofind, ret = []) { + /* + * Finds all keys in a nested object and returns them as an array + */ + var value + Object.keys(object).forEach(function(key) { + if (key === keytofind) { + ret.push(object[key]) + return true + } + if (object[key] && typeof object[key] === 'object') { + value = findKeyRecursive(object[key], keytofind, ret) + return value !== undefined + } + else { + return true + } + }) + return flatten(ret) +} + +function searchInFeatures(features, query) { + /* + * Searches for a query in an array of features + * Attributes must be in _attributes + */ + + var results = [] + var reCat = new RegExp(query.category) + var reType = new RegExp(query.type) + var reSub = new RegExp(query.subtype) + + for(var idx = 0; idx < features.length; idx++){ + var match_category = reCat.test(features[idx]._attributes.category) + var match_type = reType.test(features[idx]._attributes.type) + var match_subtype = reSub.test(features[idx]._attributes.subtype) + + if (match_category && match_type && match_subtype) { + results.push(features[idx]) + } + } + return results +} + +function transformFilename (filename, url = false) { + /* + * Transforms a full filepath: + * from example_1.xml to example 1 + * URL: from example_1.xml to example_1 + */ + filename = filename.slice(0, -4) + + if (!url) { + filename = filename.replace(/_/g, ' ') + } + + return filename +} + +function searchInObject (obj, query) { + /* + * Search for query in a fiven flatobject + */ + var results = [] + + Object.keys(obj).forEach(function (key) { + + // Attributes + if (typeof query.attribute !== "undefined"){ + var re_attr = new RegExp('_attributes.' + query.attribute) + if (!re_attr.test(key)){ + return + } + } + // Values + if (typeof query.value !== "undefined"){ + var re_valu = new RegExp(query.value) + if (!re_valu.test(obj[key])){ + return + } + } + // Elements + if (typeof query.element !== "undefined"){ + var re_elem = new RegExp('[^_]' + query.element, 'g') + if (!re_elem.test(key)){ + return + } + } + // Content + if (typeof query.content !== "undefined"){ + var re_cont = new RegExp(query.content) + if (!re_cont.test(obj[key])){ + return + } + } + + // Add hit to results as object + res = {} + res[key] = obj[key] + results.push(res) + }) + + return results +} + +function getXMLasJSON (filepath) { + /* + * TODO: This is duplicate Code. Needs to go + * TODO: Maybe check if XML is valid? + * Reads XML file and transforms it to JSON + */ + var data = {} + try { + var xml = fs.readFileSync(filepath, 'utf8') + data = convert.xml2js(xml, {compact: true}) + } catch (err) { + console.log('Error while loading ' + filepath) + console.log(err) + return data + } + return data +} + +function countReferences (features) { + /* + * Count the references in a feature + */ + var count = 0 + for (idx = 0; idx < features.length; idx++) { + count = count + features[idx]._attributes.ref.trim().split(" ").length + } + + return count +} + +module.exports = { + getXMLasJSON: function (filepath) { + /* + * Reads XML file and transforms it to JSON + */ + var data = {} + try { + var xml = fs.readFileSync(filepath, 'utf8') + data = convert.xml2js(xml, {compact: true}) + } catch (err) { + console.log('Error while loading ' + filepath) + console.log(err) + return data + } + return data + }, + + listLetters:function (file, params) { + /* + * Callback function + * Transforms a given parsed XML file into the list view + */ + return { + link: '/letters/' + file.slice(0, -4), + name: transformFilename(file), + filepath: path.join(dir, file), + url: '/api/letters/' + transformFilename(file, true) + } + }, + searchFeatures: function(file, params) { + /* + * Callback function + * Search for features in file + */ + var query = params.query + var result = undefined + var letterJS = getXMLasJSON(path.join(dir, file)) + var features = findKeyRecursive(letterJS, 'feature') + + if (typeof query.category == "undefined"){ query.category = '.*' } + if (typeof query.type == "undefined"){ query.type = '.*' } + if (typeof query.subtype == "undefined"){ query.subtype = '.*' } + + var res = searchInFeatures(features, query) + var count = countReferences(res) + + if(res.length > 0) { + result = { + collection: letterJS.TEI.teiHeader.fileDesc.titleStmt.collection._text, + count: count, + name: transformFilename(file), + link: '/letters/' + file.slice(0, -4), + url: '/api/letters/' + transformFilename(file, true), + results: res + } + } + return result + + }, + searchLetters: function(file, params){ + /* + * Callback function + * Search for query in file + */ + var query = params.query + var result = undefined + var letterJS = getXMLasJSON(path.join(dir, file)) + var flatobject = flat(letterJS.TEI) + var res = searchInObject(flatobject, query) + + if(res.length > 0) { + result = { + collection: letterJS.TEI.teiHeader.fileDesc.titleStmt.collection._text, + name: transformFilename(file), + link: '/letters/' + file.slice(0, -4), + url: '/api/letters/' + transformFilename(file, true), + results: res + } + } + return result + }, + + forAllFiles: function (callback, filepath, params={}) { + /* + * Load files from filepath and perform callback on each with given parameters + */ + var ret = [] + + try { + var files = fs.readdirSync(filepath) + } catch (err) { + console.log('> Error while loading files from ' + filepath) + console.log(err) + return [] + } + + for (var i in files) { + // We just want the XML documents + if (!files[i].endsWith('.xml')) { + continue + } + + // Call callback function on each file + var val = callback(files[i], params) + if (typeof val !== "undefined"){ + ret.push(val) + } + } + return ret + } +} diff --git a/package.json b/package.json index 1965df5..4a57fbc 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,13 @@ "dev": "node build/dev-server.js", "start": "npm run dev", "build": "node build/build.js", - "lint": "eslint --ext .js,.vue src" + "lint": "eslint --ext .js,.vue src api.js lib.js" }, "dependencies": { "express": "^4.16.2", "flat": "^4.0.0", "flatten": "^1.0.2", + "memory-cache": "^0.2.0", "vue": "^2.5.2", "vue-resource": "^1.3.4", "vue-router": "^3.0.1", diff --git a/src/components/Features.vue b/src/components/Features.vue index e1ee774..39ebd95 100644 --- a/src/components/Features.vue +++ b/src/components/Features.vue @@ -130,7 +130,7 @@ export default { // Remove undefined values Object.keys(params).forEach((key) => (params[key] == null) && delete params[key]) this.searching = true - this.$http.get('http://localhost:3000/api/feature/', {params: params}).then(function (data) { + this.$http.get('http://localhost:3000/api/features/', {params: params}).then(function (data) { this.results = data.body this.searching = false }) -- GitLab