From 788c1fb852d4406a24224f4afe64fef1e6f3b3e2 Mon Sep 17 00:00:00 2001 From: Emad aghayi Date: Mon, 27 May 2019 14:42:45 -0400 Subject: [PATCH 01/20] add logs to firebase for fetching, submitting Review and Implementation microtasks --- util/firebase_service.js | 32 +++++++++++++++++++++++++++++++- util/microtask_service.js | 23 +++++++++++++++++------ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/util/firebase_service.js b/util/firebase_service.js index 7d51763d5..f86920019 100644 --- a/util/firebase_service.js +++ b/util/firebase_service.js @@ -1852,12 +1852,42 @@ module.exports = function (AdminFirebase, Q) { eventType: event_type, eventDescription: event_description, timestamp: new Date().toLocaleTimeString("en-us", timeOptions), - timeInMicros: now('micro') + timeInMicros: now('micro'), + epochTime:{'.sv': 'timestamp'} }; var path = 'Projects/' + project_id + '/history/events'; var created_child = root_ref.child(path).push(event_schema); return created_child.key; }, + /* Add an event in the history, it stores log for fetching, submitting review and implementation microtasks + param project id text + param artifact type text + param artifact id text + param artifact name text + param event type text + param event description + return event ID text + */ + createLogEvent: function (project_id, event_type, event_description, artifact_type, artifact_id,worker_id, microtask_id,microtask_object,function_object_previous_version,function_object) { + + let event_schema = { + artifactType: artifact_type, + artifactId: artifact_id, + eventType: event_type, + eventDescription: event_description, + timestamp: new Date().toLocaleTimeString("en-us", timeOptions), + timeInMicros: now('micro'), + epochTime:{'.sv': 'timestamp'}, + workerId: worker_id, + microtaskId:microtask_id, + microtaskObject:microtask_object ? microtask_object:null, + functionObjectPreviousVersion: JSON.parse( JSON.stringify(function_object_previous_version ? function_object_previous_version :null) ) , + functionObjectCurrentVersion:JSON.parse( JSON.stringify(function_object ? function_object: null) ) + }; + var path = 'Projects/' + project_id + '/history/logs'; + var created_child = root_ref.child(path).push(event_schema); + return created_child.key; + }, /* ---------------- End Event API ------------- */ diff --git a/util/microtask_service.js b/util/microtask_service.js index 8e7ae7ccb..1c93375cd 100644 --- a/util/microtask_service.js +++ b/util/microtask_service.js @@ -9,6 +9,7 @@ module.exports = function (FirebaseService, Q) { returns promise */ function loadProject(project_id) { + if (Projects.has(project_id) === false) { Projects.set(project_id, new Map()); var Project = Projects.get(project_id); @@ -418,7 +419,7 @@ module.exports = function (FirebaseService, Q) { var implementationQ = Project.get('implementationQ'); var func = functions.get(function_id); // if function is third party api, it does not need to create implementation task - if(func.isThirdPartyAPI == true){ + if (func.isThirdPartyAPI == true) { return true; } var isDependentsComplete = true; @@ -569,6 +570,7 @@ module.exports = function (FirebaseService, Q) { completed_task.push(microtask_id); var microtask_object = microtasks.get(microtask_id); + var functionObjectPreviousVersion = JSON.parse(JSON.stringify(microtask_object)) ; microtask_object.code = funct.code ? funct.code : "defaultCode"; microtask_object.header = funct.header; microtask_object.tests = microtask_tests.tests; @@ -596,6 +598,7 @@ module.exports = function (FirebaseService, Q) { Projects.set(project_id, Project); firebase.backupState(project_id, Project); firebase.createEvent(project_id, "Implementation.Submitted", "Implementation task: " + microtask_id + " is submitted by worker: " + worker_id, "Implementation", microtask_id); + firebase.createLogEvent(project_id, "Implementation Submitted", "Implementation task: " + microtask_id + " is submitted by worker: " + worker_id, "Implementation", microtask_id,worker_id,microtask_id,microtask_object,functionObjectPreviousVersion,null); return deferred.promise; } else { @@ -651,6 +654,8 @@ module.exports = function (FirebaseService, Q) { //update function object and tests var function_object = functions.get(function_id); + //copy functino object for storing in log + var function_object_previous_version = JSON.parse(JSON.stringify(function_object)); function_object.version = implementation_object.functionVersion + 1; function_object.code = implementation_object.code; var test_set = implementation_object.tests; @@ -667,7 +672,7 @@ module.exports = function (FirebaseService, Q) { var submission_object = implementation_object.submission; if (submission_object.hasOwnProperty('requestedFunctions') && submission_object.requestedFunctions !== "null") { submission_object.requestedFunctions.forEach(function (func) { - var dependent_id = firebase.createFunction(project_id, func.name, "null", func.description, "{\n\t//Implementation code here \n\treturn \n}", func.returnType, func.parameters, "null", "null", "null", "null", false,false); + var dependent_id = firebase.createFunction(project_id, func.name, "null", func.description, "{\n\t//Implementation code here \n\treturn \n}", func.returnType, func.parameters, "null", "null", "null", "null", false, false); var dependent_object = { name: func.name, header: "null", @@ -757,7 +762,8 @@ module.exports = function (FirebaseService, Q) { }).catch(function (err) { deferred.reject(err); }); - + // log microtask, previous version, current version + firebase.createLogEvent(project_id, "Review Submitted", "Review task: " + microtask_id + " is submitted by worker: " + worker_id, "Review", microtask_id, worker_id, microtask_id, microtask_object, function_object_previous_version, function_object); }).catch(function (err) { deferred.reject(err); }); @@ -832,7 +838,7 @@ module.exports = function (FirebaseService, Q) { //Check if tasks that are already skipped available or if review tasks are not for implementation by the worker (it should be reviewed by others) reviewQ.forEach(function (id) { var review_object = microtasks.get(id); - if (skipped_task.indexOf(id) >= 0 || (review_object!= null && completed_task.indexOf(review_object.reference_id) >= 0)) { + if (skipped_task.indexOf(id) >= 0 || (review_object != null && completed_task.indexOf(review_object.reference_id) >= 0)) { review_skipped++; } }); @@ -888,7 +894,7 @@ module.exports = function (FirebaseService, Q) { } fetch_time = new Date().getTime(); //If the worker already has as on going task - }else { + } else { microtask_id = assigned_task.get('id'); microtask_type = assigned_task.get('type'); fetch_time = assigned_task.get('fetch_time'); @@ -917,6 +923,8 @@ module.exports = function (FirebaseService, Q) { inProgress_task.set('assigned_time', new Date().getTime()); inProgressQ.set(microtask_id, inProgress_task); + // log microtask, previous version + firebase.createLogEvent(project_id, "Fetch Microtask", "Fetch Microtask: " + microtask_id + " is fetched by worker: " + worker_id, microtask_type, microtask_id, worker_id, microtask_id, return_object,funct,null); //Calls a function after 10 minutes to check if the task is still with the same worker, if so removes it and put the task back in queue setTimeout(unassignLockedMicrotask, 600000, project_id, microtask_id, worker_id); } else { @@ -935,8 +943,9 @@ module.exports = function (FirebaseService, Q) { Project.set('inProgress', inProgressQ) Projects.set(project_id, Project); firebase.backupState(project_id, Project); + return return_object; - }else { + } else { console.log('Reloading project at fetch'); var load_project_promise = loadProject(project_id); load_project_promise.then(function () { @@ -1023,6 +1032,8 @@ module.exports = function (FirebaseService, Q) { Project.set('inProgressQ', inProgressQ); Projects.set(project_id, Project); firebase.backupState(project_id, Project); + // log microtask, previous version + firebase.createLogEvent(project_id, "Skipped Microtask", "Skipped Microtask: " + microtask_id + " is skipped by worker: " + worker_id, microtask_type, microtask_id, worker_id, microtask_id,null ,null,null); return; } else { From 5a9c75f7506faa1c4419209509895918106d26e0 Mon Sep 17 00:00:00 2001 From: Emad aghayi Date: Wed, 29 May 2019 12:49:25 -0400 Subject: [PATCH 02/20] add log for landing to dashboard add log for logging successfully change timestamp to firebase.serverTime.timestamp --- app.js | 149 +++++++++++++++++++++----------------- public/welcome.html | 18 ++++- util/firebase_service.js | 13 ++-- util/microtask_service.js | 22 +++--- 4 files changed, 115 insertions(+), 87 deletions(-) diff --git a/app.js b/app.js index 3cc3f39b0..dd971197a 100644 --- a/app.js +++ b/app.js @@ -6,8 +6,9 @@ const favicon = require('serve-favicon'); const logger = require('morgan'); const cookieParser = require('cookie-parser'); const bodyParser = require('body-parser'); +const firebaseConstant = require('firebase'); const cors = require('cors')({ - origin: true + origin: true }); const status = require('http-status'); const wagner = require('wagner-core'); @@ -22,67 +23,79 @@ app.set('views', path.join(__dirname, 'public')); app.use(logger('dev')); app.use(cookieParser()); app.use(session({ - name: 'server-session-cookie-id', - secret: 'my express secret', - saveUninitialized: true, - resave: true, - store: new FileStore({ - retries: 10, - minTimeout: 100, - maxTimeout: 200, - path: path.join(__dirname,'sessions') - }) + name: 'server-session-cookie-id', + secret: 'my express secret', + saveUninitialized: true, + resave: true, + store: new FileStore({ + retries: 10, + minTimeout: 100, + maxTimeout: 200, + path: path.join(__dirname, 'sessions') + }) })); -const isAuth = wagner.invoke(function() { - return (req, res, next) => { - if(req.session.user || req.path==='/login' || req.path === '/loggedin.html') { - req.user = req.session.user; - next(); - } else { - res.redirect('/login'); - } - }; +const isAuth = wagner.invoke(function () { + return (req, res, next) => { + if (req.session.user || req.path === '/login' || req.path === '/loggedin.html') { + req.user = req.session.user; + next(); + } else { + res.redirect('/login'); + } + }; }); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ - extended: false + extended: false })); -app.post('/authenticate', wagner.invoke(function(UserService, AdminFirebase, FirebaseService) { - return function(req, res) { - //var idToken = req.body.idToken; - var idToken = req.headers['authorization'].split(' ').pop(); - AdminFirebase.auth().verifyIdToken(idToken) - .then(function(decodedToken) { - var uid = decodedToken.uid; - req.session.user = decodedToken; - UserService.getUserById(uid) - .then(function(userRecord) { - // See the UserRecord reference doc for the contents of userRecord. - console.log("Successfully fetched user data:", userRecord.toJSON()); - var worker_id = userRecord.uid; - var worker_name = userRecord.displayName; - var avatar_url = userRecord.photoURL; - var firebase = FirebaseService; - var workers_list_promise = firebase.retrieveWorkersList(); - workers_list_promise.then(function(workers_list) { - if (workers_list.indexOf(worker_id) < 0) { - firebase.createWorker(worker_id, worker_name, avatar_url); - } - }).catch(function(err) { - console.trace(err); - }); - }) - .catch(function(err) { - console.log("Error fetching user data:", err); - }); - res.json({ - 'Success': 200 - }) - }).catch(function(err) { - // Handle error - console.trace(err); - }); - }; +app.post('/authenticate', wagner.invoke(function (UserService, AdminFirebase, FirebaseService) { + return function (req, res) { + //var idToken = req.body.idToken; + var idToken = req.headers['authorization'].split(' ').pop(); + AdminFirebase.auth().verifyIdToken(idToken) + .then(function (decodedToken) { + var uid = decodedToken.uid; + req.session.user = decodedToken; + UserService.getUserById(uid) + .then(function (userRecord) { + // See the UserRecord reference doc for the contents of userRecord. + console.log("Successfully fetched user data:", userRecord.toJSON()); + var worker_id = userRecord.uid; + var worker_name = userRecord.displayName; + var avatar_url = userRecord.photoURL; + var firebase = FirebaseService; + // logs worker authentication + var ref = AdminFirebase.database().ref().child('Workers').child('logs'); + ref.push({ + eventType: "LoggedIn", + eventDescription: "Logged in successfully", + timestamp: firebaseConstant.database.ServerValue.TIMESTAMP, + workerId: userRecord.uid, + workerName: userRecord.displayName + }).catch(e => { + console.log(e) + }); + + var workers_list_promise = firebase.retrieveWorkersList(); + workers_list_promise.then(function (workers_list) { + if (workers_list.indexOf(worker_id) < 0) { + firebase.createWorker(worker_id, worker_name, avatar_url); + } + }).catch(function (err) { + console.trace(err); + }); + }) + .catch(function (err) { + console.log("Error fetching user data:", err); + }); + res.json({ + 'Success': 200 + }) + }).catch(function (err) { + // Handle error + console.trace(err); + }); + }; })); app.use(cookieParser()); app.use(isAuth); @@ -90,21 +103,21 @@ app.use('/api/v1', require('./routes/api')(wagner)); app.use(express.static(path.join(__dirname, 'public'))); app.use(require('./routes/app-routing')(wagner)); // catch 404 and forward to error handler -app.use(function(req, res, next) { - var err = new Error('Not Found'); - err.status = 404; - next(err); +app.use(function (req, res, next) { + var err = new Error('Not Found'); + err.status = 404; + next(err); }); // error handler -app.use(function(err, req, res, next) { - // set locals, only providing error in development - res.locals.message = err.message; - res.locals.error = req.app.get('env') === 'development' ? err : {}; +app.use(function (err, req, res, next) { + // set locals, only providing error in development + res.locals.message = err.message; + res.locals.error = req.app.get('env') === 'development' ? err : {}; - // render the error page - res.status(err.status || 500); - res.sendFile(path.join(__dirname, '/public/404.html')); + // render the error page + res.status(err.status || 500); + res.sendFile(path.join(__dirname, '/public/404.html')); }); module.exports = app; diff --git a/public/welcome.html b/public/welcome.html index 2b11e71ba..e7e71d405 100644 --- a/public/welcome.html +++ b/public/welcome.html @@ -303,7 +303,8 @@

If you are ready, please insert the name project in the bellow input field t
- +
@@ -323,7 +324,6 @@

If you are ready, please insert the name project in the bellow input field t -