From 0080ac5f1444124bfc8afa9b65d1d6c8ea49683f Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 3 Sep 2020 11:03:02 -0500 Subject: [PATCH 1/3] Detect and apply library updates. Closes #1884 --- src/common/updates/Updates.js | 54 +++++++++++-- src/plugins/ApplyUpdates/ApplyUpdates.js | 27 +++++-- src/plugins/CheckUpdates/CheckUpdates.js | 11 +-- src/plugins/ImportLibrary/ImportLibrary.js | 75 ++++++++++--------- .../ForgeActionButton/Libraries.json.ejs | 1 + .../panels/Sidebar/SidebarPanel.js | 9 +-- utils/extender.js | 40 +++++----- 7 files changed, 127 insertions(+), 90 deletions(-) diff --git a/src/common/updates/Updates.js b/src/common/updates/Updates.js index bb59a8dc1..f6bde9c13 100644 --- a/src/common/updates/Updates.js +++ b/src/common/updates/Updates.js @@ -4,14 +4,15 @@ define([ 'deepforge/viz/PlotlyDescExtractor', 'deepforge/viz/FigureExtractor', './Version', - 'q' + 'text!deepforge/extensions/Libraries.json', ], function( Storage, PlotlyDescExtractor, FigureExtractor, Version, - Q + LibrariesTxt, ) { + const Libraries = JSON.parse(LibrariesTxt); const GRAPH = 'Graph'; const getGraphNodes = async function(core, rootNode, graphNodes=[]) { const children = await core.loadChildren(rootNode); @@ -148,12 +149,48 @@ define([ const Updates = {}; - Updates.getAvailableUpdates = function(core, rootNode) { - return Q.all(allUpdates.map(update => update.isNeeded(core, rootNode))) - .then(isNeeded => { - const updates = allUpdates.filter((update, i) => isNeeded[i]); - return updates; - }); + Updates.getAvailableUpdates = async function(core, rootNode) { + const migrations = await this.getMigrationUpdates(core, rootNode); + const libUpdates = await this.getLibraryUpdates(core, rootNode); + return migrations.concat(libUpdates); + }; + + Updates.getMigrationUpdates = async function(core, rootNode) { + const isNeeded = await Promise.all( + allUpdates.map(update => update.isNeeded(core, rootNode)) + ); + const updates = allUpdates.filter((update, i) => isNeeded[i]); + return updates.map(update => ({ + type: Updates.MIGRATION, + name: update.name, + })); + }; + + Updates.getLibraryUpdates = async function(core, rootNode) { + const children = await core.loadChildren(rootNode); + const libraryType = Object.values(core.getAllMetaNodes(rootNode)) + .find(node => core.getAttribute(node, 'name') === 'LibraryCode'); + const libraries = children.filter(child => core.isTypeOf(child, libraryType)); + const libUpdates = Libraries.filter(libraryInfo => { + const {name, version} = libraryInfo; + const libNode = libraries + .find(library => { + const nodeName = core.getAttribute(library, 'name'); + return nodeName === name || nodeName === `${name}InitCode`; + }); + + if (libNode) { + const nodeVersion = core.getAttribute(libNode, 'version'); + const installedVersion = new Version(nodeVersion || '0.0.0'); + const availableVersion = new Version(version || '0.0.0'); + return installedVersion.lessThan(availableVersion); + } + }); + return libUpdates.map(libInfo => ({ + type: Updates.LIBRARY, + name: `Update ${libInfo.name} library`, + info: libInfo, + })); }; Updates.getUpdates = function(names) { @@ -170,5 +207,6 @@ define([ // Constants Updates.MIGRATION = 'Migration'; Updates.SEED = 'SeedUpdate'; + Updates.LIBRARY = 'Library'; return Updates; }); diff --git a/src/plugins/ApplyUpdates/ApplyUpdates.js b/src/plugins/ApplyUpdates/ApplyUpdates.js index e95e56de4..10c45939c 100644 --- a/src/plugins/ApplyUpdates/ApplyUpdates.js +++ b/src/plugins/ApplyUpdates/ApplyUpdates.js @@ -2,13 +2,15 @@ /*eslint-env node, browser*/ define([ + 'plugin/ImportLibrary/ImportLibrary/ImportLibrary', 'deepforge/updates/Updates', 'text!./metadata.json', - 'plugin/PluginBase' + 'underscore', ], function ( + PluginBase, Updates, pluginMetadata, - PluginBase + _, ) { 'use strict'; @@ -50,24 +52,33 @@ define([ ApplyUpdates.prototype.main = async function (callback) { // Retrieve the updates to apply const config = this.getCurrentConfig(); - const updateNames = config.updates || []; + const {updates=[]} = config; - if (!updateNames.length) { + if (!updates.length) { this.result.setSuccess(true); return callback(null, this.result); } // Apply each of the updates - const updates = Updates.getUpdates(updateNames); + const [migrations, libUpdates] = _.partition( + updates, + update => update.type === Updates.MIGRATION + ); - for (let i = 0, len = updates.length; i < len; i++) { - const update = updates[i]; + for (let i = 0, len = migrations.length; i < len; i++) { + const update = migrations[i]; this.logger.info(`Applying update: ${update.name} to ${this.projectId}`); await update.apply(this.core, this.rootNode, this.META); } + for (let i = libUpdates.length; i--;) { + const libraryInfo = libUpdates[i].info; + await this.importLibrary(libraryInfo); + } + // Save the project - await this.save(`Applied project updates: ${updateNames.join(",")}`); + const updateNames = updates.map(update => update.name); + await this.save(`Applied project updates: ${updateNames.join(',')}`); this.result.setSuccess(true); callback(null, this.result); diff --git a/src/plugins/CheckUpdates/CheckUpdates.js b/src/plugins/CheckUpdates/CheckUpdates.js index 3a1c42ec0..5871d90a6 100644 --- a/src/plugins/CheckUpdates/CheckUpdates.js +++ b/src/plugins/CheckUpdates/CheckUpdates.js @@ -72,16 +72,7 @@ define([ this.logger.info(`Updates available for ${this.projectId}: ${updateNames}`); // Combine and report the result - const msgs = seedUpdates - .concat( - updates.map(update => { - return { - type: Updates.MIGRATION, - node: null, - name: update.name - }; - }) - ); + const msgs = seedUpdates.concat(updates); msgs.forEach(msg => { const {node} = msg; diff --git a/src/plugins/ImportLibrary/ImportLibrary.js b/src/plugins/ImportLibrary/ImportLibrary.js index 2ddb5f05f..e8716d7aa 100644 --- a/src/plugins/ImportLibrary/ImportLibrary.js +++ b/src/plugins/ImportLibrary/ImportLibrary.js @@ -46,24 +46,21 @@ define([ * * @param {function(string, plugin.PluginResult)} callback - the result callback */ - ImportLibrary.prototype.main = function (callback) { + ImportLibrary.prototype.main = async function () { const config = this.getCurrentConfig(); const libraryInfo = config.libraryInfo; - return this.addSeedToBranch(libraryInfo.seed) - .then(branchName => this.createGMELibraryFromBranch(branchName, libraryInfo)) - .then(branchInfo => this.removeTemporaryBranch(branchInfo)) - .then(() => this.updateMetaForLibrary(libraryInfo)) - .then(() => this.addLibraryInitCode(libraryInfo)) - .then(() => this.save(`Imported ${libraryInfo.name} library`)) - .then(() => { - this.result.setSuccess(true); - callback(null, this.result); - }) - .catch(err => { - this.logger.error(`Could not check the libraries: ${err}`); - callback(err, this.result); - }); + await this.importLibrary(libraryInfo); + await this.save(`Imported ${libraryInfo.name} library`); + this.result.setSuccess(true); + }; + + ImportLibrary.prototype.importLibrary = async function (libraryInfo) { + const branchName = await this.addSeedToBranch(libraryInfo.seed); + const branchInfo = await this.createGMELibraryFromBranch(branchName, libraryInfo); + await this.removeTemporaryBranch(branchInfo); + await this.updateMetaForLibrary(libraryInfo); + await this.addLibraryInitCode(libraryInfo); }; ImportLibrary.prototype.getUniqueBranchName = function (basename) { @@ -105,7 +102,12 @@ define([ let rootHash = commit.root; libraryData.commitHash = commit._id; - await this.core.addLibrary(this.rootNode, name, rootHash, libraryData); + const alreadyExists = this.core.getLibraryNames(this.rootNode).includes(name); + if (alreadyExists) { + await this.core.updateLibrary(this.rootNode, name, rootHash, libraryData); + } else { + await this.core.addLibrary(this.rootNode, name, rootHash, libraryData); + } return { name: branchName, hash: commit._id @@ -116,7 +118,7 @@ define([ return this.project.deleteBranch(branch.name, branch.hash); }; - ImportLibrary.prototype.updateMetaForLibrary = function (libraryInfo) { + ImportLibrary.prototype.updateMetaForLibrary = async function (libraryInfo) { const nodeNames = libraryInfo.nodeTypes; const libraryNodes = this.getLibraryMetaNodes(libraryInfo.name); @@ -132,34 +134,37 @@ define([ .filter(node => !!node); // Add containment relationships to the meta - return this.core.loadChildren(this.rootNode) - .then(children => { - let parent = children.find(node => this.core.getAttribute(node, 'name') === 'MyResources'); - if (!parent) throw new Error('Could not find resources location'); - nodes.forEach(node => this.core.setChildMeta(parent, node)); - }); + const children = await this.core.loadChildren(this.rootNode); + let parent = children.find(node => this.core.getAttribute(node, 'name') === 'MyResources'); + if (!parent) throw new Error('Could not find resources location'); + nodes.forEach(node => this.core.setChildMeta(parent, node)); }; - ImportLibrary.prototype.addLibraryInitCode = function (libraryInfo) { - // Get the library fco node - // Add the initialization code for this library; + ImportLibrary.prototype.addLibraryInitCode = async function (libraryInfo) { const libraryNodes = this.getLibraryMetaNodes(libraryInfo.name); - const LibraryCode = this.getLibraryCodeNode(); - const FCO = this.getFCONode(); - - // Make the LibraryCode node - const node = this.core.createNode({ - parent: this.rootNode, - base: LibraryCode - }); + const node = await this.createLibraryCodeNode(libraryInfo.name); this.core.setAttribute(node, 'code', libraryInfo.initCode || ''); - this.core.setAttribute(node, 'name', `${libraryInfo.name}InitCode`); + this.core.setAttribute(node, 'name', libraryInfo.name); + this.core.setAttribute(node, 'version', libraryInfo.version); + const FCO = this.getFCONode(); const libraryFCO = libraryNodes .find(node => this.core.getPointerPath(node, 'base') === this.core.getPath(FCO)); this.core.setPointer(node, 'library', libraryFCO); + return node; + }; + + ImportLibrary.prototype.createLibraryCodeNode = async function (libName) { + const LibraryCode = this.getLibraryCodeNode(); + const libraryCodeNodes = (await this.core.loadChildren(this.rootNode)) + .filter(node => this.core.isTypeOf(node, LibraryCode)); + + return libraryCodeNodes.find(node => { + const name = this.core.getAttribute(node, 'name'); + return name === libName || name === `${libName}InitCode`; + }) || this.core.createNode({parent: this.rootNode, base: LibraryCode}); }; ImportLibrary.prototype.getLibraryMetaNodes = function (libraryName) { diff --git a/src/visualizers/panels/ForgeActionButton/Libraries.json.ejs b/src/visualizers/panels/ForgeActionButton/Libraries.json.ejs index 4a22ab414..fe487b522 100644 --- a/src/visualizers/panels/ForgeActionButton/Libraries.json.ejs +++ b/src/visualizers/panels/ForgeActionButton/Libraries.json.ejs @@ -1,6 +1,7 @@ <%= JSON.stringify(extensions.map(function(ext) { return { name: ext.name, + version: ext.version, description: ext.description, nodeTypes: ext.nodeTypes, initCode: ext.initCode, diff --git a/src/visualizers/panels/Sidebar/SidebarPanel.js b/src/visualizers/panels/Sidebar/SidebarPanel.js index 89f8b0aad..3e0f1c68b 100644 --- a/src/visualizers/panels/Sidebar/SidebarPanel.js +++ b/src/visualizers/panels/Sidebar/SidebarPanel.js @@ -173,7 +173,8 @@ define([ if (update.type === Updates.SEED) { return 0; } - if (Updates.getUpdate(update.name).beforeLibraryUpdates) { + const hasBeforeUpdates = update.type === Updates.MIGRATION && Updates.getUpdate(update.name).beforeLibraryUpdates; + if (hasBeforeUpdates) { return 1; } return 2; @@ -195,13 +196,9 @@ define([ SidebarPanel.prototype.applyMigrationUpdates = async function (updates) { const pluginId = 'ApplyUpdates'; - const names = updates.map(update => update.name); const context = this._client.getCurrentPluginContext(pluginId); - context.pluginConfig = { - updates: names, - }; - + context.pluginConfig = {updates}; await Q.ninvoke(this._client, 'runServerPlugin', pluginId, context); }; diff --git a/utils/extender.js b/utils/extender.js index 3d9a093e9..a83bed7e5 100644 --- a/utils/extender.js +++ b/utils/extender.js @@ -6,6 +6,7 @@ // const path = require('path'); const fs = require('fs'); +const fsp = require('fs').promises; const pacote = require('pacote'); const rm_rf = require('rimraf'); const exists = require('exists-file'); @@ -67,7 +68,7 @@ extender.getInstalledConfigType = function(name) { extender.install = async function(projectName, isReinstall) { await exec(`npm install ${projectName}`); - const {name} = await pacote.manifest(projectName); + const {name, version} = await pacote.manifest(projectName); const extRoot = path.join(__dirname, '..', 'node_modules', name); // Check for the extensions.json in the project (look up type, etc) @@ -85,15 +86,16 @@ extender.install = async function(projectName, isReinstall) { } try { - extConfig = JSON.parse(fs.readFileSync(extConfigPath, 'utf8')); + extConfig = JSON.parse(await fsp.readFile(extConfigPath, 'utf8')); + extConfig.version = version; } catch(e) { // Invalid JSON - throw `Invalid ${extender.EXT_CONFIG_NAME}: ${e}`; + throw new Error(`Invalid ${extender.EXT_CONFIG_NAME}: ${e}`); } // Try to add the extension to the project (using the extender) extType = extConfig.type; if (!extender.isSupportedType(extType)) { - throw `Unrecognized extension type: "${extType}"`; + throw new Error(`Unrecognized extension type: "${extType}"`); } // add project info to the config let project = { @@ -115,7 +117,7 @@ extender.install = async function(projectName, isReinstall) { } allExtConfigs[extType][extConfig.name] = extConfig; - const config = await extender.install[extType](extConfig, project, !!isReinstall); + const config = await extender.install[extType](extConfig, project, isReinstall); extConfig = config || extConfig; // Update the deployment config allExtConfigs[extType][extConfig.name] = extConfig; @@ -206,13 +208,6 @@ function makeInstallFor(typeCfg) { }; } -//var PLUGIN_ROOT = path.join(__dirname, '..', 'src', 'plugins', 'Export'); -//makeInstallFor({ - //type: 'Export:Pipeline', - //template: path.join(PLUGIN_ROOT, 'format.js.ejs'), - //targetDir: path.join(PLUGIN_ROOT, 'formats', '<%=name%>') -//}); - const LIBRARY_ROOT = path.join(__dirname, '..', 'src', 'visualizers', 'panels', 'ForgeActionButton'); makeInstallFor({ @@ -226,17 +221,16 @@ makeInstallFor({ const libraryType = 'Library'; const LIBRARY_TEMPLATE_PATH = path.join(__dirname, '..', 'src', 'visualizers', 'panels', 'ForgeActionButton', 'Libraries.json.ejs'); -extender.install[libraryType] = (config, project/*, isReinstall*/) => { - return webgme.all.import(project.arg) // import the seed and stuff - .then(() => { - // Add the initCode to the config - config.initCode = config.initCode || ''; - if (config.initCode) { - const initCodePath = path.join(project.root, config.initCode); - config.initCode = fs.readFileSync(initCodePath, 'utf8'); - } - return updateTemplateFile(LIBRARY_TEMPLATE_PATH, libraryType); - }); +extender.install[libraryType] = async (config, project/*, isReinstall*/) => { + await webgme.all.import(project.arg); // import the seed and stuff + // Add the initCode to the config + console.log(config); + config.initCode = config.initCode || ''; + if (config.initCode) { + const initCodePath = path.join(project.root, config.initCode); + config.initCode = await fsp.readFile(initCodePath, 'utf8'); + } + return updateTemplateFile(LIBRARY_TEMPLATE_PATH, libraryType); }; extender.uninstall[libraryType] = (/*name, config*/) => { From e00ea931b51e3963727be62c866ac47a76291905 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 3 Sep 2020 14:05:43 -0500 Subject: [PATCH 2/3] Move libraries file --- .../ForgeActionButton => common/extensions}/Libraries.json | 0 .../extensions}/Libraries.json.ejs | 0 utils/extender.js | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{visualizers/panels/ForgeActionButton => common/extensions}/Libraries.json (100%) rename src/{visualizers/panels/ForgeActionButton => common/extensions}/Libraries.json.ejs (100%) diff --git a/src/visualizers/panels/ForgeActionButton/Libraries.json b/src/common/extensions/Libraries.json similarity index 100% rename from src/visualizers/panels/ForgeActionButton/Libraries.json rename to src/common/extensions/Libraries.json diff --git a/src/visualizers/panels/ForgeActionButton/Libraries.json.ejs b/src/common/extensions/Libraries.json.ejs similarity index 100% rename from src/visualizers/panels/ForgeActionButton/Libraries.json.ejs rename to src/common/extensions/Libraries.json.ejs diff --git a/utils/extender.js b/utils/extender.js index a83bed7e5..7fed7225b 100644 --- a/utils/extender.js +++ b/utils/extender.js @@ -219,8 +219,8 @@ makeInstallFor({ // Add the extension type for another domain/library const libraryType = 'Library'; -const LIBRARY_TEMPLATE_PATH = path.join(__dirname, '..', 'src', 'visualizers', - 'panels', 'ForgeActionButton', 'Libraries.json.ejs'); +const LIBRARY_TEMPLATE_PATH = path.join(__dirname, '..', 'src', 'common', + 'extensions', 'Libraries.json.ejs'); extender.install[libraryType] = async (config, project/*, isReinstall*/) => { await webgme.all.import(project.arg); // import the seed and stuff // Add the initCode to the config From 215c82095cf4cbcf6b63417c0c74c0c05d53b57d Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 3 Sep 2020 14:07:18 -0500 Subject: [PATCH 3/3] Fix references to Libraries.json --- src/visualizers/panels/ForgeActionButton/LibraryDialog.js | 2 +- .../OperationInterfaceEditorControl.EventHandlers.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/visualizers/panels/ForgeActionButton/LibraryDialog.js b/src/visualizers/panels/ForgeActionButton/LibraryDialog.js index defcff215..cdbbc06b9 100644 --- a/src/visualizers/panels/ForgeActionButton/LibraryDialog.js +++ b/src/visualizers/panels/ForgeActionButton/LibraryDialog.js @@ -4,7 +4,7 @@ define([ 'common/storage/constants', 'q', 'underscore', - 'text!./Libraries.json', + 'text!deepforge/extensions/Libraries.json', 'text!./LibraryDialogModal.html', 'css!./LibraryDialog.css' ], function( diff --git a/src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js b/src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js index f76e5e541..324761b6b 100644 --- a/src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js +++ b/src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js @@ -4,7 +4,7 @@ define([ 'deepforge/OperationCode', 'deepforge/Constants', './Colors', - 'text!panels/ForgeActionButton/Libraries.json', + 'text!deepforge/extensions/Libraries.json', ], function( EasyDAGControlEventHandlers, OperationCode,