diff --git a/.arclint b/.arclint index 7fb4c0f..9540bf8 100644 --- a/.arclint +++ b/.arclint @@ -1,87 +1,88 @@ { "exclude": [ "(^externals/)", "(^webroot/rsrc/externals/(?!javelin/))", "(/__tests__/data/)", "(^support/aphlict/server/package-lock.json)" ], "linters": { "chmod": { "type": "chmod" }, "filename": { "type": "filename" }, "generated": { "type": "generated" }, "javelin": { "type": "javelin", "include": "(\\.js$)", "exclude": [ "(^support/aphlict/)" ] }, "jshint-browser": { "type": "jshint", "include": "(\\.js$)", "exclude": [ "(^support/aphlict/server/.*\\.js$)", "(^webroot/rsrc/externals/javelin/core/init_node\\.js$)" ], "jshint.jshintrc": "support/lint/browser.jshintrc" }, "jshint-node": { "type": "jshint", "include": [ "(^support/aphlict/server/.*\\.js$)", "(^webroot/rsrc/externals/javelin/core/init_node\\.js$)" ], "jshint.jshintrc": "support/lint/node.jshintrc" }, "json": { "type": "json", "include": [ "(^src/docs/book/.*\\.book$)", "(^support/lint/jshintrc$)", "(^\\.arcconfig$)", "(^\\.arclint$)", "(\\.json$)" ] }, "merge-conflict": { "type": "merge-conflict" }, "nolint": { "type": "nolint" }, "phutil-library": { "type": "phutil-library", "include": "(\\.php$)" }, "spelling": { "type": "spelling" }, "text": { "type": "text", "exclude": [ - "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))" + "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))", + "(^docs/examples.*json)" ] }, "text-without-length": { "type": "text", "include": [ "(^src/(.*/)?__tests__/[^/]+/.*\\.(txt|json|expect))" ], "severity": { "3": "disabled" } }, "xhpast": { "type": "xhpast", "include": "(\\.php$)", "standard": "phutil.xhpast", "xhpast.php-version": "5.5" } } } diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d9a10c0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/README.md b/README.md new file mode 100644 index 0000000..29282a1 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +Semi-Structured - Application for managing evolving datasets +============================================================ + +This Phorge extension is designed to allow storing and managing objects that +are not fully defined, and allowing the definition to easily evolve over time. + +Data consists of "Instance" and "Types" (classes). Each object in an instance of +a single Type. Types define the required data (fields) of their instances. + +Instance and Types have all the features of other Phorge objects - comments, +mentions, notifications, transactions, search and policies. + +Objects have +- ID +- Title and Free text +- Free, structured data (JSON blob, without a schema) +- Custom fields, defined in the Class. These are typed, may have validation, + and may be searchable and may hold references to other Phorge objects. + + + +Custom fields are defined using Phorge's +[Custom Fields](https://we.phorge.it/book/phorge/article/custom_fields/) +mechanism, which is quite capable. If needed, Advanced Custom Fields may be +added by writing PHP code. + + + +------------ + + +Installation: + +- `git clone` this repository somewhere safe +- In Phorge's `conf/local/local.json` add the path to the `src/` dir to the + entry `load-libraries`, something like this: + +``` + ... + "load-libraries": [ + "/somewhere/safe/semi-structured/src/" + ], + ... +``` + +- Run Phorge's `bin/storage upgrade` and restart Phorge. + + +In `docs/examples/xkcd` you'll find a script that loads some example data: +``` +$ ./docs/examples/xkcd/load-xkcd.py +``` diff --git a/docs/examples/xkcd/load-xkcd.py b/docs/examples/xkcd/load-xkcd.py new file mode 100755 index 0000000..b0c10d2 --- /dev/null +++ b/docs/examples/xkcd/load-xkcd.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +import argparse +import json +import subprocess +import sys +from os.path import dirname +from pprint import pprint +from typing import Optional + +parser = argparse.ArgumentParser( + description="Load reference data into a new Object Type." +) +parser.add_argument( + "phorge_uri", + help='URI to use for the Phorge server, like "https://we.phorge.it/"', +) + + +def setup_server(uri): + global phorge_uri + phorge_uri = uri + + +def read_file(filename: str) -> str: + path = dirname(__file__) + + with open(path + "/" + filename, "rt") as input: + return input.read() + + +def arc(*command, input: Optional[str] = None): + return subprocess.check_output( + [ + "arc", + f"--conduit-uri={phorge_uri}", + *command, + ], + input=input, + text=True, + ) + + +def call_conduit(method, input: dict): + input_raw = json.dumps(input) + result = arc("call-conduit", "--", method, input=input_raw) + return json.loads(result) + + +def main(): + args = parser.parse_args() + + if "we.phorge.it" in args.phorge_uri: + print( + "Don't use public servers to load this example data " + "- deploy to your own server!" + ) + sys.exit(3) + + setup_server(args.phorge_uri) + + # Prep Object Type + + custom_data = read_file("xkcd.config.json") + + create_type_xactions = [ + { + "type": "name", + "value": "XKCD Comics", + }, + { + "type": "description", + "value": "Demo data set about romance, sarcasm, math, and language.", + }, + { + "type": "customfieldsdef", + # The value here would be a json-encoded string containing json. + "value": custom_data, + }, + ] + + result = call_conduit( + "semistructured.type.edit", {"transactions": create_type_xactions} + ) + + if result["error"]: + if ( + 'method "semistructured.type.edit" does not exist.' + in result["errorMessage"] + ): + print("The extension is not installed!") + sys.exit(4) + + print("Error creating type!") + pprint(result) + sys.exit(5) + + new_type = result["response"]["object"] + id = new_type["id"] + class_phid = new_type["phid"] + print(f"New Object Type created! Id {id}, phid {class_phid}") + print(f"Visit it here: {phorge_uri}/semistruct/type/{id}/".replace("//", "/")) + + # Now let's load some objects... + data_items = json.loads(read_file("xkcd.data.json")) + + for item in data_items: + print("Creating instance " + item["name"] + "...") + + xactions = [{"type": "classPHID", "value": class_phid}] + + for key in ["name", "description", "rawdata"]: + xactions.append( + { + "type": key, + "value": item[key], + } + ) + + result = call_conduit( + "semistructured.instance.edit", {"transactions": xactions} + ) + + custom_fields = [ + "custom.xkcd:id", + "custom.xkcd:alttext", + "custom.xkcd:explainLink", + "custom.xkcd:comicType", + ] + + if result["error"]: + print("Error creating instance!") + pprint(result) + sys.exit(6) + + item_phid = result["response"]["object"]["phid"] + print(f"Created {item_phid} {item['name']} - now adding custom fields...") + + xactions = list() + for key in custom_fields: + xactions.append( + { + "type": key, + "value": item[key], + } + ) + + result = call_conduit( + "semistructured.instance.edit", + {"transactions": xactions, "objectIdentifier": item_phid}, + ) + + if result["error"]: + print("Error updating instance!") + pprint(result) + sys.exit(7) + + print("Success") + + +if __name__ == "__main__": + main() diff --git a/docs/examples/xkcd/xkcd.config.json b/docs/examples/xkcd/xkcd.config.json new file mode 100644 index 0000000..6b66aec --- /dev/null +++ b/docs/examples/xkcd/xkcd.config.json @@ -0,0 +1,30 @@ +{ + "xkcd:id": { + "name": "XKCD number", + "type": "int", + "caption": "Running number assigned by Munroe", + "required": true, + "search": true + }, + "xkcd:comicType": { + "name": "Item Type", + "type": "select", + "search": true, + "options": { + "unknown": "Unknown", + "simple": "Simple Image", + "other": "Something Fancy" + } + }, + "xkcd:alttext": { + "name": "Alt text", + "caption": "Text that shows up when hovering the mouse over the image", + "type": "text", + "search": true, + "fulltext": true + }, + "xkcd:explainLink": { + "type": "link", + "name": "ExplainXKCD url" + } +} diff --git a/docs/examples/xkcd/xkcd.data.json b/docs/examples/xkcd/xkcd.data.json new file mode 100644 index 0000000..2ad5f33 --- /dev/null +++ b/docs/examples/xkcd/xkcd.data.json @@ -0,0 +1,47 @@ +[ + { + "name": "Garden", + "description": "Each observer is getting their very own instance of a Garden, that grows very slowly over time.", + "custom.xkcd:id": 1663, + "custom.xkcd:comicType": "other", + "custom.xkcd:explainLink": "https://www.explainxkcd.com/wiki/index.php/1663:_Garden", + "custom.xkcd:alttext": "Relax.", + "rawdata": "{\n\"cause\": \"April Fools 2016\"\n\n}" + }, + { + "name": "Fermat's First Theorem", + "description": "", + "custom.xkcd:id": 2689, + "custom.xkcd:comicType": "simple", + "custom.xkcd:explainLink": "https://www.explainxkcd.com/wiki/index.php/2689:_Fermat%27s_First_Theorem", + "custom.xkcd:alttext": "Mathematicians quickly determined that it spells ANT BNECN, an unusual theoretical dish which was not successfully cooked until Andrew Wiles made it for breakfast in the 1990s", + "rawdata": "{}" + }, + { + "name": "Escape Speed", + "description": "This one is a a game with a spaceship.", + "custom.xkcd:id": 2765, + "custom.xkcd:comicType": "other", + "custom.xkcd:alttext": "Gotta go fast.", + "custom.xkcd:explainLink": "https://www.explainxkcd.com/wiki/index.php/2765:_Escape_Speed", + "rawdata": "{\n\"status\": \"broken\"\n\n\n}" + }, + { + "name": "Cosmologist on a Tire Swing", + "description": "", + "custom.xkcd:id": 1352, + "custom.xkcd:comicType": "simple", + "custom.xkcd:explainLink": "https://www.explainxkcd.com/wiki/index.php/1352:_Cosmologist_on_a_Tire_Swing", + "custom.xkcd:alttext": "No matter how fast I swing, I can never travel outside this loop! Maybe space outside it doesn't exist! But I bet it does. This tire came from somewhere.", + "rawdata": "{\n\"image url\": \"https://imgs.xkcd.com/comics/cosmologist_on_a_tire_swing.png\"\n}" + }, + { + "name": "Fireflies", + "description": "", + "custom.xkcd:id": 2802, + "custom.xkcd:comicType": "simple", + "custom.xkcd:explainLink": "https://www.explainxkcd.com/wiki/index.php/2802:_Fireflies", + "custom.xkcd:alttext": "I feel bad for Earth 2 and their shadowflies.", + "rawdata": "{\n\"imageUrl\": \"https://imgs.xkcd.com/comics/fireflies.png\"\n}" + } +] \ No newline at end of file