{ "cells": [ { "cell_type": "markdown", "id": "dacba8a3-91fe-45ad-af7b-f5082466b969", "metadata": {}, "source": [ "# Basic JSON file parsing\n", "## Info gathered by the scientist on eLabFTW\n", "### Experiment 45 \"NEW PLD Deposition Layer\"\n", "#### General info\n", "* Date and time of creation\n", "* Category\n", "* Full name of the scientist\n", "* Related items (sample, PLD target)\n", "\n", "#### Instrument\n", "* Chamber (by ID)\n", "* Laser system\n", "* RHEED system\n", "\n", "#### Process\n", "* Sample (by ID)\n", "* Layer progressive number\n", "* Target (by ID)\n", "* Heater temperature\n", "* Heater target distance\n", "* Buffer gas\n", "* Process pressure\n", "* Heating method\n", "* Laser intensity\n", "* Duration\n", "* Repetition rate\n", "* Thickness\n", "\n", "#### Post annealing\n", "* Buffer gas used in PA\n", "* Process pressure of PA\n", "* Heater temperature of PA\n", "* Duration of PA\n", "\n", "### Chamber\n", "\n", "### Sample\n", "\n", "### Target" ] }, { "cell_type": "markdown", "id": "c6321d97-4c3e-4e73-a3a2-e3f23ae0a733", "metadata": {}, "source": [ "## Brick by brick\n", "Let's start by loading and printing the contents of Experiment 45's JSON as downloaded from eLabFTW." ] }, { "cell_type": "code", "execution_count": 1, "id": "dfa122ad-6de1-4282-bb67-8fcd877e6678", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"access_key\": null,\n", " \"body\": \"\",\n", " \"body_html\": \"\",\n", " \"canread\": \"{\\\"base\\\": 40, \\\"teams\\\": [], \\\"users\\\": [], \\\"teamgroups\\\": []}\",\n", " \"canread_is_immutable\": 0,\n", " \"canwrite\": \"{\\\"base\\\": 20, \\\"teams\\\": [], \\\"users\\\": [], \\\"teamgroups\\\": []}\",\n", " \"canwrite_is_immutable\": 0,\n", " \"category\": 2,\n", " \"category_color\": \"8b8d43\",\n", " \"category_title\": \"Deposition\",\n", " \"comments\": [],\n", " \"compounds\": [],\n", " \"containers\": [],\n", " \"content_type\": 1,\n", " \"created_at\": \"2026-01-20 16:11:32\",\n", " \"custom_id\": null,\n", " \"date\": \"2026-01-20\",\n", " \"elabid\": \"20260120-b72ca9659e21e904f5dbd12b2625a007ed97ed57\",\n", " \"events_start\": null,\n", " \"events_start_itemid\": null,\n", " \"exclusive_edit_mode\": null,\n", " \"experiments_links\": [],\n", " \"firstname\": \"Emiliano\",\n", " \"fullname\": \"Emiliano Di Gennaro\",\n", " \"id\": 45,\n", " \"is_pinned\": 0,\n", " \"items_links\": [\n", " {\n", " \"entityid\": 854,\n", " \"title\": \"LAO_single_crystal_01\",\n", " \"custom_id\": null,\n", " \"elabid\": \"20260107-1f8d0a6d9cf61be826b8b35f5516959848815310\",\n", " \"link_state\": 1,\n", " \"is_bookable\": 0,\n", " \"page\": \"database.php\",\n", " \"type\": \"items\",\n", " \"category_title\": \"NEW_PLD Target\",\n", " \"category_color\": \"29aeb9\",\n", " \"status_title\": \"Available\",\n", " \"status_color\": \"6a7753\"\n", " },\n", " {\n", " \"entityid\": 855,\n", " \"title\": \"Na-26-001\",\n", " \"custom_id\": null,\n", " \"elabid\": \"20260107-e83642d2b806e5db5ebb0d6309d874f4b4461114\",\n", " \"link_state\": 1,\n", " \"is_bookable\": 0,\n", " \"page\": \"database.php\",\n", " \"type\": \"items\",\n", " \"category_title\": \"NEW_Sample\",\n", " \"category_color\": \"29aeb9\",\n", " \"status_title\": \"Available\",\n", " \"status_color\": \"6a7753\"\n", " }\n", " ],\n", " \"lastchangeby\": 2,\n", " \"lastname\": \"Di Gennaro\",\n", " \"locked\": 0,\n", " \"locked_at\": null,\n", " \"lockedby\": null,\n", " \"metadata\": \"{\\\"elabftw\\\": {\\\"extra_fields_groups\\\": [{\\\"id\\\": 4, \\\"name\\\": \\\"Process\\\"}, {\\\"id\\\": 7, \\\"name\\\": \\\"Laser\\\"}, {\\\"id\\\": 8, \\\"name\\\": \\\"Pre Annealing\\\"}, {\\\"id\\\": 6, \\\"name\\\": \\\"Post Annealing\\\"}, {\\\"id\\\": 3, \\\"name\\\": \\\"Instruments\\\"}]}, \\\"extra_fields\\\": {\\\"Sample\\\": {\\\"type\\\": \\\"items\\\", \\\"value\\\": 855, \\\"group_id\\\": 4, \\\"position\\\": 0}, \\\"Target\\\": {\\\"type\\\": \\\"items\\\", \\\"value\\\": 854, \\\"group_id\\\": 4, \\\"position\\\": 2, \\\"required\\\": true}, \\\"Chamber\\\": {\\\"type\\\": \\\"items\\\", \\\"value\\\": 72, \\\"group_id\\\": 3, \\\"position\\\": 0}, \\\"Duration\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"s\\\", \\\"units\\\": [\\\"s\\\", \\\"min\\\"], \\\"value\\\": \\\"340\\\", \\\"group_id\\\": 4, \\\"position\\\": 5}, \\\"Spot Area\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"mm^2\\\", \\\"units\\\": [\\\"mm^2\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 7, \\\"position\\\": 1}, \\\"Thickness\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"u.c.\\\", \\\"units\\\": [\\\"u.c.\\\", \\\"s\\\"], \\\"value\\\": \\\"10\\\", \\\"group_id\\\": 4, \\\"position\\\": 6}, \\\"Buffer gas\\\": {\\\"type\\\": \\\"select\\\", \\\"value\\\": \\\"O2\\\", \\\"options\\\": [\\\"O2\\\", \\\"N2\\\", \\\"Ar\\\", \\\"\\\"], \\\"group_id\\\": 4, \\\"position\\\": 3}, \\\"Duration PA\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"s\\\", \\\"units\\\": [\\\"s\\\", \\\"min\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 6}, \\\"Duration Pre\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"s\\\", \\\"units\\\": [\\\"s\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 8}, \\\"Laser System\\\": {\\\"type\\\": \\\"text\\\", \\\"value\\\": \\\"Excimer \\\", \\\"group_id\\\": 3}, \\\"RHEED System\\\": {\\\"type\\\": \\\"text\\\", \\\"value\\\": \\\"staib\\\", \\\"group_id\\\": 3}, \\\"Buffer gas PA\\\": {\\\"type\\\": \\\"select\\\", \\\"value\\\": \\\"O2\\\", \\\"options\\\": [\\\"O2\\\", \\\"N2\\\", \\\"Ar\\\", \\\"\\\"], \\\"group_id\\\": 6, \\\"position\\\": 3}, \\\"Buffer gas Pre\\\": {\\\"type\\\": \\\"select\\\", \\\"value\\\": \\\"O2\\\", \\\"options\\\": [\\\"O2\\\", \\\"N2\\\", \\\"Ar\\\", \\\"\\\"], \\\"group_id\\\": 8, \\\"position\\\": 3}, \\\"Heating Method\\\": {\\\"type\\\": \\\"select\\\", \\\"value\\\": \\\"Radiative Heater\\\", \\\"options\\\": [\\\"Radiative Heater\\\", \\\"Laser Heater\\\"], \\\"group_id\\\": 4, \\\"position\\\": 9}, \\\"Laser Intensity\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"J/(s cm^2)\\\", \\\"units\\\": [\\\"J/(s cm^2)\\\"], \\\"value\\\": \\\"1.5\\\", \\\"group_id\\\": 7, \\\"position\\\": 0}, \\\"Repetition rate\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"Hz\\\", \\\"units\\\": [\\\"Hz\\\"], \\\"value\\\": \\\"1\\\", \\\"group_id\\\": 7, \\\"position\\\": 4}, \\\"Process pressure \\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"mbar\\\", \\\"units\\\": [\\\"mbar\\\"], \\\"value\\\": \\\"1e-3\\\", \\\"group_id\\\": 4, \\\"position\\\": 4}, \\\"Heater temperature \\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"\\u00b0C\\\", \\\"units\\\": [\\\"\\u00b0C\\\"], \\\"value\\\": \\\"760\\\", \\\"group_id\\\": 4, \\\"position\\\": 7}, \\\"Process pressure PA\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"mbar\\\", \\\"units\\\": [\\\"mbar\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 6, \\\"position\\\": 4}, \\\"Process pressure Pre\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"mbar\\\", \\\"units\\\": [\\\"mbar\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 8, \\\"position\\\": 4}, \\\"Heater temperature PA\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"\\u00b0C\\\", \\\"units\\\": [\\\"\\u00b0C\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 6}, \\\"Laser Rastering Speed\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"\\\", \\\"units\\\": [], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 7, \\\"position\\\": 3}, \\\"Heater temperature Pre\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"\\u00b0C\\\", \\\"units\\\": [\\\"\\u00b0C\\\"], \\\"value\\\": \\\"\\\", \\\"group_id\\\": 8}, \\\"Heater-target distance\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"mm\\\", \\\"units\\\": [\\\"mm\\\"], \\\"value\\\": \\\"38\\\", \\\"group_id\\\": 4, \\\"position\\\": 8}, \\\"Laser Rastering Geometry\\\": {\\\"type\\\": \\\"select\\\", \\\"value\\\": \\\"none\\\", \\\"options\\\": [\\\"none\\\", \\\"on a square\\\", \\\"on a rectangle\\\", \\\"on a line\\\", \\\"other\\\"], \\\"group_id\\\": 7, \\\"position\\\": 2}, \\\"Layer Progressive Number\\\": {\\\"type\\\": \\\"number\\\", \\\"unit\\\": \\\"\\\", \\\"units\\\": [], \\\"value\\\": \\\"1\\\", \\\"group_id\\\": 4, \\\"position\\\": 1, \\\"required\\\": true}}}\",\n", " \"metadata_decoded\": {\n", " \"elabftw\": {\n", " \"extra_fields_groups\": [\n", " {\n", " \"id\": 4,\n", " \"name\": \"Process\"\n", " },\n", " {\n", " \"id\": 7,\n", " \"name\": \"Laser\"\n", " },\n", " {\n", " \"id\": 8,\n", " \"name\": \"Pre Annealing\"\n", " },\n", " {\n", " \"id\": 6,\n", " \"name\": \"Post Annealing\"\n", " },\n", " {\n", " \"id\": 3,\n", " \"name\": \"Instruments\"\n", " }\n", " ]\n", " },\n", " \"extra_fields\": {\n", " \"Sample\": {\n", " \"type\": \"items\",\n", " \"value\": 855,\n", " \"group_id\": 4,\n", " \"position\": 0\n", " },\n", " \"Target\": {\n", " \"type\": \"items\",\n", " \"value\": 854,\n", " \"group_id\": 4,\n", " \"position\": 2,\n", " \"required\": true\n", " },\n", " \"Chamber\": {\n", " \"type\": \"items\",\n", " \"value\": 72,\n", " \"group_id\": 3,\n", " \"position\": 0\n", " },\n", " \"Duration\": {\n", " \"type\": \"number\",\n", " \"unit\": \"s\",\n", " \"units\": [\n", " \"s\",\n", " \"min\"\n", " ],\n", " \"value\": \"340\",\n", " \"group_id\": 4,\n", " \"position\": 5\n", " },\n", " \"Spot Area\": {\n", " \"type\": \"number\",\n", " \"unit\": \"mm^2\",\n", " \"units\": [\n", " \"mm^2\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 7,\n", " \"position\": 1\n", " },\n", " \"Thickness\": {\n", " \"type\": \"number\",\n", " \"unit\": \"u.c.\",\n", " \"units\": [\n", " \"u.c.\",\n", " \"s\"\n", " ],\n", " \"value\": \"10\",\n", " \"group_id\": 4,\n", " \"position\": 6\n", " },\n", " \"Buffer gas\": {\n", " \"type\": \"select\",\n", " \"value\": \"O2\",\n", " \"options\": [\n", " \"O2\",\n", " \"N2\",\n", " \"Ar\",\n", " \"\"\n", " ],\n", " \"group_id\": 4,\n", " \"position\": 3\n", " },\n", " \"Duration PA\": {\n", " \"type\": \"number\",\n", " \"unit\": \"s\",\n", " \"units\": [\n", " \"s\",\n", " \"min\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 6\n", " },\n", " \"Duration Pre\": {\n", " \"type\": \"number\",\n", " \"unit\": \"s\",\n", " \"units\": [\n", " \"s\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 8\n", " },\n", " \"Laser System\": {\n", " \"type\": \"text\",\n", " \"value\": \"Excimer \",\n", " \"group_id\": 3\n", " },\n", " \"RHEED System\": {\n", " \"type\": \"text\",\n", " \"value\": \"staib\",\n", " \"group_id\": 3\n", " },\n", " \"Buffer gas PA\": {\n", " \"type\": \"select\",\n", " \"value\": \"O2\",\n", " \"options\": [\n", " \"O2\",\n", " \"N2\",\n", " \"Ar\",\n", " \"\"\n", " ],\n", " \"group_id\": 6,\n", " \"position\": 3\n", " },\n", " \"Buffer gas Pre\": {\n", " \"type\": \"select\",\n", " \"value\": \"O2\",\n", " \"options\": [\n", " \"O2\",\n", " \"N2\",\n", " \"Ar\",\n", " \"\"\n", " ],\n", " \"group_id\": 8,\n", " \"position\": 3\n", " },\n", " \"Heating Method\": {\n", " \"type\": \"select\",\n", " \"value\": \"Radiative Heater\",\n", " \"options\": [\n", " \"Radiative Heater\",\n", " \"Laser Heater\"\n", " ],\n", " \"group_id\": 4,\n", " \"position\": 9\n", " },\n", " \"Laser Intensity\": {\n", " \"type\": \"number\",\n", " \"unit\": \"J/(s cm^2)\",\n", " \"units\": [\n", " \"J/(s cm^2)\"\n", " ],\n", " \"value\": \"1.5\",\n", " \"group_id\": 7,\n", " \"position\": 0\n", " },\n", " \"Repetition rate\": {\n", " \"type\": \"number\",\n", " \"unit\": \"Hz\",\n", " \"units\": [\n", " \"Hz\"\n", " ],\n", " \"value\": \"1\",\n", " \"group_id\": 7,\n", " \"position\": 4\n", " },\n", " \"Process pressure \": {\n", " \"type\": \"number\",\n", " \"unit\": \"mbar\",\n", " \"units\": [\n", " \"mbar\"\n", " ],\n", " \"value\": \"1e-3\",\n", " \"group_id\": 4,\n", " \"position\": 4\n", " },\n", " \"Heater temperature \": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"760\",\n", " \"group_id\": 4,\n", " \"position\": 7\n", " },\n", " \"Process pressure PA\": {\n", " \"type\": \"number\",\n", " \"unit\": \"mbar\",\n", " \"units\": [\n", " \"mbar\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 6,\n", " \"position\": 4\n", " },\n", " \"Process pressure Pre\": {\n", " \"type\": \"number\",\n", " \"unit\": \"mbar\",\n", " \"units\": [\n", " \"mbar\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 8,\n", " \"position\": 4\n", " },\n", " \"Heater temperature PA\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 6\n", " },\n", " \"Laser Rastering Speed\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\",\n", " \"units\": [],\n", " \"value\": \"\",\n", " \"group_id\": 7,\n", " \"position\": 3\n", " },\n", " \"Heater temperature Pre\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"\",\n", " \"group_id\": 8\n", " },\n", " \"Heater-target distance\": {\n", " \"type\": \"number\",\n", " \"unit\": \"mm\",\n", " \"units\": [\n", " \"mm\"\n", " ],\n", " \"value\": \"38\",\n", " \"group_id\": 4,\n", " \"position\": 8\n", " },\n", " \"Laser Rastering Geometry\": {\n", " \"type\": \"select\",\n", " \"value\": \"none\",\n", " \"options\": [\n", " \"none\",\n", " \"on a square\",\n", " \"on a rectangle\",\n", " \"on a line\",\n", " \"other\"\n", " ],\n", " \"group_id\": 7,\n", " \"position\": 2\n", " },\n", " \"Layer Progressive Number\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\",\n", " \"units\": [],\n", " \"value\": \"1\",\n", " \"group_id\": 4,\n", " \"position\": 1,\n", " \"required\": true\n", " }\n", " }\n", " },\n", " \"modified_at\": \"2026-01-20 16:17:59\",\n", " \"next_step\": \"add process data\",\n", " \"orcid\": \"0000-0003-4231-9776\",\n", " \"page\": \"experiments\",\n", " \"rating\": 0,\n", " \"recent_comment\": null,\n", " \"related_experiments_links\": [\n", " {\n", " \"entityid\": 46,\n", " \"title\": \"Na-26-001 deposition test II\",\n", " \"custom_id\": null,\n", " \"link_state\": 1,\n", " \"page\": \"experiments.php\",\n", " \"type\": \"experiments\",\n", " \"category_title\": \"Deposition\",\n", " \"category_color\": \"8b8d43\",\n", " \"status_title\": null,\n", " \"status_color\": null\n", " }\n", " ],\n", " \"related_items_links\": [],\n", " \"sharelink\": \"https://elabftw.fisica.unina.it:8080/experiments.php?mode=view&id=45\",\n", " \"state\": 1,\n", " \"status\": null,\n", " \"status_color\": null,\n", " \"status_title\": null,\n", " \"steps\": [\n", " {\n", " \"id\": 35,\n", " \"item_id\": 45,\n", " \"body\": \"add process data\",\n", " \"ordering\": 1,\n", " \"finished\": 0,\n", " \"finished_time\": null,\n", " \"deadline\": null,\n", " \"deadline_notif\": 0\n", " },\n", " {\n", " \"id\": 36,\n", " \"item_id\": 45,\n", " \"body\": \"add RHEED data\",\n", " \"ordering\": 2,\n", " \"finished\": 0,\n", " \"finished_time\": null,\n", " \"deadline\": null,\n", " \"deadline_notif\": 0\n", " },\n", " {\n", " \"id\": 37,\n", " \"item_id\": 45,\n", " \"body\": \"add RHEED images\",\n", " \"ordering\": 3,\n", " \"finished\": 0,\n", " \"finished_time\": null,\n", " \"deadline\": null,\n", " \"deadline_notif\": 0\n", " }\n", " ],\n", " \"tags\": null,\n", " \"tags_id\": null,\n", " \"team\": 1,\n", " \"team_name\": \"Default team\",\n", " \"timestamped\": 0,\n", " \"timestamped_at\": null,\n", " \"timestampedby\": null,\n", " \"title\": \"Na-26-001 deposition test I\",\n", " \"type\": \"experiments\",\n", " \"uploads\": [],\n", " \"userid\": 2\n", "}\n" ] } ], "source": [ "import json\n", "\n", "with open(\"../tests/objects/experiment_45_elab.json\", \"r\") as f:\n", " x = json.load(f)\n", " print(json.dumps(x,indent=2))\n", " f.close()" ] }, { "cell_type": "markdown", "id": "a2e857a5-efdb-4177-a5cc-063270985531", "metadata": {}, "source": [ "For testing purposes now we'll create and print a simplified dictionary containing a sample of the harvested data for each group of metadata." ] }, { "cell_type": "code", "execution_count": 2, "id": "dae4b791-fd08-4f41-ba42-82edcf4e3cde", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"instrument\": {\n", " \"deposition_chamber\": 72,\n", " \"laser_system\": \"Excimer \",\n", " \"rheed_system\": \"staib\"\n", " },\n", " \"multilayer\": {\n", " \"layer_1\": {\n", " \"operator\": \"Emiliano Di Gennaro\",\n", " \"sample\": 855,\n", " \"temperature\": \"760\",\n", " \"target\": 854\n", " }\n", " }\n", "}\n" ] } ], "source": [ "with open(\"../tests/objects/experiment_45_elab.json\", \"r\") as f:\n", " rawdata = json.load(f)\n", " extra = rawdata[\"metadata_decoded\"][\"extra_fields\"]\n", " ordered = {\n", " \"instrument\": {\n", " \"deposition_chamber\": extra[\"Chamber\"][\"value\"], # ID of associated resource (PLD chamber) - useless as is!\n", " \"laser_system\": extra[\"Laser System\"][\"value\"],\n", " \"rheed_system\": extra[\"RHEED System\"][\"value\"]\n", " },\n", " \"multilayer\": {\n", " \"layer_1\": {\n", " \"operator\": rawdata[\"fullname\"],\n", " \"sample\": extra[\"Sample\"][\"value\"], # ID of associated sample - useless as is!\n", " \"temperature\": extra[\"Heater temperature \"][\"value\"], # space at the end is a config error in eLab!\n", " \"target\": extra[\"Target\"][\"value\"] # ID of associated resource (PLD target) - useless as is!\n", " }\n", " },\n", " }\n", " print(json.dumps(ordered,indent=2))\n", " f.close()" ] }, { "cell_type": "markdown", "id": "c9992a47-ce3c-47ec-94bd-26fbec020962", "metadata": {}, "source": [ "Some issues rise here:\n", "* First of all the fields \"deposition_chamber\", \"sample\" and \"target\" **refer to the value of the eLabFTW ID of the associated resource** which is useless as is since it does not contain any relevant data on these objects;\n", "* Second, the same sample can have two different eLab Experiments associated to it, each representing **a different layer** of the deposition.\n", "\n", "> Note: a layer progressive number is tracked by the scientist, and it can be found in the JSON dictionary under `metadata_decoded -> extra_fields -> Layer Progressive Number -> value`.\n", "\n", "### Multiple layers from known sources\n", "One problem at a time: first of all I can create an \"ordered\" dictionary with an empty \"multilayer\" key and append the layer-specific value of every layer later using the *dict().update()* method." ] }, { "cell_type": "code", "execution_count": 3, "id": "b3afa42e-e982-4dd3-9ea6-cf7918dc276f", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"instrument\": {\n", " \"deposition_chamber\": 72,\n", " \"laser_system\": \"Excimer \",\n", " \"rheed_system\": \"staib\"\n", " },\n", " \"multilayer\": {\n", " \"layer_1\": {\n", " \"operator\": \"Emiliano Di Gennaro\",\n", " \"sample\": {\n", " \"type\": \"items\",\n", " \"value\": 855,\n", " \"group_id\": 4,\n", " \"position\": 0\n", " },\n", " \"temperature\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"760\",\n", " \"group_id\": 4,\n", " \"position\": 7\n", " },\n", " \"target\": {\n", " \"type\": \"items\",\n", " \"value\": 854,\n", " \"group_id\": 4,\n", " \"position\": 2,\n", " \"required\": true\n", " }\n", " }\n", " }\n", "}\n" ] } ], "source": [ "with open(\"../tests/objects/experiment_45_elab.json\", \"r\") as f:\n", " rawdata = json.load(f)\n", " extra = rawdata[\"metadata_decoded\"][\"extra_fields\"]\n", " layers = {\n", " \"layer_1\": {\n", " \"operator\": rawdata[\"fullname\"],\n", " \"sample\": extra[\"Sample\"], # ID of associated sample - useless as is!\n", " \"temperature\": extra[\"Heater temperature \"], # space at the end is a config error in eLab!\n", " \"target\": extra[\"Target\"]\n", " }\n", " }\n", "\n", " ordered = {\n", " \"instrument\": {\n", " \"deposition_chamber\": extra[\"Chamber\"][\"value\"], # ID of associated resource (PLD chamber) - useless as is!\n", " \"laser_system\": extra[\"Laser System\"][\"value\"],\n", " \"rheed_system\": extra[\"RHEED System\"][\"value\"]\n", " },\n", " \"multilayer\": {\n", " },\n", " }\n", " for l in layers:\n", " ordered[\"multilayer\"].update(\n", " {l: layers[l]}\n", " )\n", " print(json.dumps(ordered, indent=2))\n", " f.close()" ] }, { "cell_type": "markdown", "id": "4a7ff14f-d2fc-4485-a174-a23248791a6f", "metadata": {}, "source": [ "Now entering the second layer: Experiment 46.\n", "\n", "If I were to create a \"layers\" dictionary with the same info from the two different experiments it would look like this:" ] }, { "cell_type": "code", "execution_count": 4, "id": "9212afba-9868-467f-ac4d-8cbce0f1537a", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"layer_1\": {\n", " \"operator\": \"Emiliano Di Gennaro\",\n", " \"sample\": {\n", " \"type\": \"items\",\n", " \"value\": 855,\n", " \"group_id\": 4,\n", " \"position\": 0\n", " },\n", " \"temperature\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"500\",\n", " \"group_id\": 4,\n", " \"position\": 7\n", " },\n", " \"target\": {\n", " \"type\": \"items\",\n", " \"value\": 466,\n", " \"group_id\": 4,\n", " \"position\": 2,\n", " \"required\": true\n", " }\n", " },\n", " \"layer_2\": \"\",\n", " \"layer_0\": {\n", " \"operator\": \"Emiliano Di Gennaro\",\n", " \"sample\": {\n", " \"type\": \"items\",\n", " \"value\": 855,\n", " \"group_id\": 4,\n", " \"position\": 0\n", " },\n", " \"temperature\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"760\",\n", " \"group_id\": 4,\n", " \"position\": 7\n", " },\n", " \"target\": {\n", " \"type\": \"items\",\n", " \"value\": 854,\n", " \"group_id\": 4,\n", " \"position\": 2,\n", " \"required\": true\n", " }\n", " }\n", "}\n" ] } ], "source": [ "with open(\"../tests/objects/experiment_45_elab.json\", \"r\") as L01file, open(\"../tests/objects/experiment_46_elab.json\", \"r\") as L02file:\n", " raw01 = json.load(L01file)\n", " raw02 = json.load(L02file)\n", " layer_list = [raw01, raw02]\n", "\n", " layers = { \"layer_\" + str(index + 1) : \"\" for index in range(len(layer_list)) }\n", " for i,layer in enumerate(layer_list):\n", " extra = layer[\"metadata_decoded\"][\"extra_fields\"]\n", " layers.update({\n", " f\"layer_{i}\": {\n", " \"operator\": layer[\"fullname\"],\n", " \"sample\": extra[\"Sample\"], # ID of associated sample - useless as is!\n", " \"temperature\": extra[\"Heater temperature \"], # space at the end is a config error in eLab!\n", " \"target\": extra[\"Target\"]\n", " }\n", " })\n", "\n", " print(json.dumps(layers, indent=2))\n", "\n", " L01file.close()\n", " L02file.close()" ] }, { "cell_type": "markdown", "id": "82700735-73fb-4b0a-aa97-3072c6330a48", "metadata": {}, "source": [ "But that only works because I know exactly how many layers there are and in which order they're stored.\n", "\n", "How we're storing and downloading the experiment data related to a same sample is still subject of discussion. The parser should be able to associate different JSON files to their own sample and group the files related to the same experiment; each file has a *Layer Progressive Number* which associates the data saved in the file to a specific layer, and it's imperative that the parser:\n", "* Recognises the absence of a layer (e.g. [1, 2, 4], returns that no 3rd layer exists);\n", "* Names every layer \"layer_X\" where X is the progressive number starting from 1 (not 0).\n", "\n", "### Multiple layers from uncategorized files\n", "Supposing I don't know that files *experiment_45_elab.json* and *experiment_46_elab.json* contain data of layers 1 and 2 of the same sample NA-26-001 I can always load every file in the folder indiscriminately and:\n", "* Filter out every non-eLabFTW file (by some recognition pattern).\n", "* Group the data by the sample it's associated to.\n", "\n", "\n", "#### Filter out non-eLabFTW files using the key \"elabid\" as challenge\n", "If the key *elabid* is present in the root of a JSON file then assume the file is an eLabFTW experiment output." ] }, { "cell_type": "code", "execution_count": 5, "id": "d8b83ba0-6b5b-425a-b365-8f5fa6ab4117", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "../tests/objects/experiment_45_elab.json\n", "../tests/objects/experiment_46_elab.json\n" ] } ], "source": [ "import os\n", "\n", "elabfiles = []\n", "for filename in os.listdir(\"../tests/objects\"):\n", " if filename.endswith(\".json\"):\n", " try:\n", " with open(os.path.join(\"../tests/objects\", filename), \"r\") as f:\n", " data = json.load(f)\n", " if data.get(\"elabid\"): # insert specific NeXus requirements here later\n", " \n", " elabfiles.append(filename)\n", " f.close()\n", " except json.decoder.JSONDecodeError as e: # invalid files \"masked\" as JSON\n", " #print(f\"wait a moment: {e}\") # just for debug\n", " pass\n", "\n", "for i in elabfiles:\n", " print(os.path.join(\"../tests/objects/\", i))" ] }, { "cell_type": "markdown", "id": "825f32d1-eb8f-4f9a-af94-d7258e897f8f", "metadata": {}, "source": [ "#### Group the data by sample\n", "Lookup the value of the key \"*Sample*\" in the extra fields; two experiments with that same value are associated to the same sample. To obtain this result the best course of action is *probably* to create a dictionary with every unique sample and all the data associated with it:\n", "* The dictionary starts empty.\n", "* The parser then reads the data from the first eLab-compliant file, in particular it reads the ID of the sample associated.\n", "* If the ID (later the name) of the sample is not a key in the root of sample_dict create a new key, otherwise skip.\n", "* Add layer-specific data to new layer in the sample_dict." ] }, { "cell_type": "code", "execution_count": 6, "id": "4b7961e6-817f-44b7-b2de-16d15d9ec26a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{855: {'instrument': {'deposition_chamber': 72, 'laser_system': 'Excimer ', 'rheed_system': 'staib'}, 'multilayer': {'layer_1': {'operator': 'Emiliano Di Gennaro', 'created_at': '2026-01-20 16:11:32', 'sample': {'type': 'items', 'value': 855, 'group_id': 4, 'position': 0}, 'temperature': {'type': 'number', 'unit': '°C', 'units': ['°C'], 'value': '760', 'group_id': 4, 'position': 7}, 'target': {'type': 'items', 'value': 854, 'group_id': 4, 'position': 2, 'required': True}}, 'layer_2': {'operator': 'Emiliano Di Gennaro', 'created_at': '2026-01-20 16:18:48', 'sample': {'type': 'items', 'value': 855, 'group_id': 4, 'position': 0}, 'temperature': {'type': 'number', 'unit': '°C', 'units': ['°C'], 'value': '500', 'group_id': 4, 'position': 7}, 'target': {'type': 'items', 'value': 466, 'group_id': 4, 'position': 2, 'required': True}}}}}\n" ] } ], "source": [ "sample_dict = {}\n", "for filename in elabfiles:\n", " with open(os.path.join(\"../tests/objects/\", filename), \"r\") as f:\n", " layer = json.load(f)\n", " extra = layer[\"metadata_decoded\"][\"extra_fields\"]\n", " sample = extra[\"Sample\"][\"value\"]\n", " lpn = int(extra[\"Layer Progressive Number\"][\"value\"]) # Layer Progressive Number\n", " if not sample_dict.get(sample): # if not existent yet, initialize\n", " sample_dict[sample] = {\n", " \"instrument\": {\n", " \"deposition_chamber\": extra[\"Chamber\"][\"value\"], # ID of associated resource (PLD chamber) - useless as is!\n", " \"laser_system\": extra[\"Laser System\"][\"value\"],\n", " \"rheed_system\": extra[\"RHEED System\"][\"value\"]\n", " },\n", " \"multilayer\": {}\n", " }\n", " sample_dict[sample][\"multilayer\"][f\"layer_{lpn}\"] = {\n", " \"operator\": layer[\"fullname\"],\n", " \"created_at\": layer[\"created_at\"],\n", " \"sample\": extra[\"Sample\"], # ID of associated sample - useless as is!\n", " \"temperature\": extra[\"Heater temperature \"], # space at the end is a config error in eLab!\n", " \"target\": extra[\"Target\"]\n", " }\n", " \n", "print(sample_dict)" ] }, { "cell_type": "markdown", "id": "27f876e0-291e-43d6-8b9d-3ce533896e5e", "metadata": {}, "source": [ "#### Look out for missing layers\n" ] }, { "cell_type": "code", "execution_count": 14, "id": "6a5281dc-7fc3-4de7-845c-2bc2b54d4bb1", "metadata": {}, "outputs": [], "source": [ "#sample_dict[855][\"multilayer\"][\"layer_4\"] = {} # for debug purposes\n", "\n", "def find_missing(lst):\n", " '''\n", " Finds missing integers in unsorted list.\n", " Time complexity is NlogN but since N is at most 10^1 it's not a problem for us.\n", " Source: geekforgeeks.org.\n", " '''\n", " lst.sort() # sorts list\n", " return sorted(set(range(lst[0], lst[-1])) - set(lst))\n", "\n", "for item in sample_dict:\n", " layer_names = list(sample_dict[item].get(\"multilayer\").keys())\n", " numbers = sorted(int(layer.split('_')[1]) for layer in layer_names)\n", " missing = find_missing(numbers)\n", " if missing:\n", " print(\"Warning: some layers appear to be missing.\")\n", " print(f\"The missing layers are: \")\n", " for i in missing:\n", " print(f\"* layer_{i}\")" ] }, { "cell_type": "markdown", "id": "028ac2b1-3389-472d-ba05-de6cfc9a9fda", "metadata": {}, "source": [ "#### Find duplicates" ] }, { "cell_type": "code", "execution_count": 13, "id": "14583887-8feb-4507-a06d-ddc557c0a875", "metadata": {}, "outputs": [], "source": [ "def find_duplicates(lst): # list of integers\n", " result = []\n", " lst.sort() # sort list just in case\n", " for i in range(len(lst)-1):\n", " #print(lst[i]) # debug\n", " if lst[i] == lst[i+1]:\n", " result.append(lst[i])\n", " return sorted(set(result))\n", "\n", "for item in sample_dict:\n", " layer_names = list(sample_dict[item].get(\"multilayer\").keys())\n", " numbers = sorted(int(layer.split('_')[1]) for layer in layer_names)\n", " dupes = find_duplicates(numbers)\n", " if dupes:\n", " print(\"Warning: some layers are duplicated.\")\n", " print(f\"The duplicate layers are: \")\n", " for i in dupes:\n", " print(f\"* layer_{i}\")" ] }, { "cell_type": "markdown", "id": "e2716da5-ff75-45d1-b765-d8bfb2ecaf71", "metadata": {}, "source": [ "### Names not ID's\n", "> TO-DO: Replace ID's of eLabFTW items with their actual names (might need a working API key).\n", "\n", "Since eLab ID's are not relevant to this project we need factual information about the sample itself. For instance, we want to fetch its name, chemical formula and dimensions - all data present on the eLabFTW entry, obtainable on the *items* API endpoint using the eLab ID of the sample." ] }, { "cell_type": "code", "execution_count": 9, "id": "24793e2b-67bb-4e8b-9d93-7802d3af7fca", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdin", "output_type": "stream", "text": [ "Paste API key here: ········\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "{'access_key': None, 'available': 1, 'body': '', 'body_html': '', 'book_can_overlap': 1, 'book_cancel_minutes': 0, 'book_is_cancellable': 1, 'book_max_minutes': 0, 'book_max_slots': 0, 'book_users_can_in_past': 0, 'canbook': '{\"base\": 40, \"teams\": [], \"users\": [], \"teamgroups\": []}', 'canread': '{\"base\": 40, \"teams\": [], \"users\": [], \"teamgroups\": []}', 'canread_is_immutable': 0, 'canwrite': '{\"base\": 30, \"teams\": [], \"users\": [], \"teamgroups\": []}', 'canwrite_is_immutable': 0, 'category': 19, 'category_color': '29aeb9', 'category_title': 'NEW_Sample', 'comments': [], 'compounds': [], 'containers': [], 'content_type': 1, 'created_at': '2026-01-07 13:20:02', 'custom_id': None, 'date': '2026-01-07', 'elabid': '20260107-e83642d2b806e5db5ebb0d6309d874f4b4461114', 'events_start': None, 'events_start_itemid': None, 'exclusive_edit_mode': None, 'experiments_links': [], 'firstname': 'Emiliano', 'fullname': 'Emiliano Di Gennaro', 'id': 855, 'is_bookable': 0, 'is_pinned': 0, 'is_procurable': 0, 'items_links': [{'entityid': 853, 'title': 'test_01', 'custom_id': None, 'elabid': '20260107-8a39f3d60b8422a878f14dbc5aa1956a6b939e07', 'link_state': 1, 'is_bookable': 0, 'page': 'database.php', 'type': 'items', 'category_title': 'NEW_Substrates batch', 'category_color': '29aeb9', 'status_title': 'Available', 'status_color': '6a7753'}, {'entityid': 180, 'title': 'NFFA -01', 'custom_id': None, 'elabid': '20240422-d90c61f98b5a368b877b3c7bcdc5448612037b0e', 'link_state': 1, 'is_bookable': 0, 'page': 'database.php', 'type': 'items', 'category_title': 'Proposal', 'category_color': 'cdab8f', 'status_title': None, 'status_color': None}, {'entityid': 826, 'title': 'CART - 6', 'custom_id': None, 'elabid': '20250415-b8c364bae6a2f74be82de1d9370c96d6b031c4d4', 'link_state': 1, 'is_bookable': 0, 'page': 'database.php', 'type': 'items', 'category_title': 'Sample Position', 'category_color': '26a269', 'status_title': None, 'status_color': None}], 'lastchangeby': 11, 'lastname': 'Di Gennaro', 'locked': 0, 'locked_at': None, 'lockedby': None, 'metadata': '{\"extra_fields\": {\"Owner\": {\"type\": \"users\", \"value\": 2, \"position\": 1, \"required\": true}, \"STD-ID\": {\"type\": \"number\", \"unit\": \"\", \"units\": [], \"value\": \"26001\", \"position\": 0, \"required\": true, \"description\": \"This is an internal ID identifier\"}, \"Position\": {\"type\": \"items\", \"value\": 826, \"position\": 4}, \"Proposal\": {\"type\": \"items\", \"value\": 180, \"position\": 5}, \"Subtrate batch\": {\"type\": \"items\", \"value\": 853, \"position\": 2}, \"Substrate Holder\": {\"type\": \"text\", \"value\": \"1\", \"position\": 3}}}', 'metadata_decoded': {'extra_fields': {'Owner': {'type': 'users', 'value': 2, 'position': 1, 'required': True}, 'STD-ID': {'type': 'number', 'unit': '', 'units': [], 'value': '26001', 'position': 0, 'required': True, 'description': 'This is an internal ID identifier'}, 'Position': {'type': 'items', 'value': 826, 'position': 4}, 'Proposal': {'type': 'items', 'value': 180, 'position': 5}, 'Subtrate batch': {'type': 'items', 'value': 853, 'position': 2}, 'Substrate Holder': {'type': 'text', 'value': '1', 'position': 3}}}, 'modified_at': '2026-01-21 22:04:27', 'next_step': None, 'orcid': '0000-0003-4231-9776', 'page': 'database', 'proc_currency': 0, 'proc_pack_qty': 0, 'proc_price_notax': '0.00', 'proc_price_tax': '0.00', 'rating': 0, 'recent_comment': None, 'related_experiments_links': [{'entityid': 41, 'title': 'NEW PLD Deposition Layer', 'custom_id': None, 'link_state': 1, 'page': 'experiments.php', 'type': 'experiments', 'category_title': 'Deposition', 'category_color': '8b8d43', 'status_title': 'Running', 'status_color': '29AEB9'}, {'entityid': 43, 'title': 'NEW PLD Deposition Layer I', 'custom_id': None, 'link_state': 1, 'page': 'experiments.php', 'type': 'experiments', 'category_title': 'Deposition', 'category_color': '8b8d43', 'status_title': None, 'status_color': None}, {'entityid': 45, 'title': 'Na-26-001 deposition test I', 'custom_id': None, 'link_state': 1, 'page': 'experiments.php', 'type': 'experiments', 'category_title': 'Deposition', 'category_color': '8b8d43', 'status_title': None, 'status_color': None}, {'entityid': 46, 'title': 'Na-26-001 deposition test II', 'custom_id': None, 'link_state': 1, 'page': 'experiments.php', 'type': 'experiments', 'category_title': 'Deposition', 'category_color': '8b8d43', 'status_title': None, 'status_color': None}], 'related_items_links': [], 'sharelink': 'https://elabftw.fisica.unina.it:8080/database.php?mode=view&id=855', 'state': 1, 'status': 1, 'status_color': '6a7753', 'status_title': 'Available', 'steps': [], 'tags': None, 'tags_id': None, 'team': 1, 'team_name': 'Default team', 'timestamped': 0, 'timestamped_at': None, 'timestampedby': None, 'title': 'Na-26-001', 'type': 'items', 'uploads': [], 'userid': 2}\n" ] } ], "source": [ "import requests\n", "from getpass import getpass # not to leak my key through jupyter + git\n", "\n", "def call_sample(API_KEY, elabid, API_URL=\"https://elabftw.fisica.unina.it/\"):\n", " full_elab_url = f\"{API_URL}api/v2\" # API endpoint root for eLabFTW\n", " items_url = f\"{full_elab_url}/items\" # API endpoint /items\n", " header = {\n", " \"Authorization\": API_KEY,\n", " \"Content-Type\": \"application/json\"\n", " }\n", " sample = requests.get(\n", " headers=header,\n", " url=f\"{items_url}/{elabid}\",\n", " verify=True\n", " )\n", " return sample.json()\n", "\n", "apikey = getpass(\"Paste API key here: \")\n", "testing = call_sample(apikey, 855)\n", "print(testing)" ] }, { "cell_type": "markdown", "id": "89d89d7a-0e13-42c2-83ba-fb03d5a2c39b", "metadata": {}, "source": [ "#### Filtering data\n", "Now let's select only the useful data, which at the moment is just the name." ] }, { "cell_type": "code", "execution_count": 10, "id": "0ffa8e82-2d7e-4dae-9081-a776f1e5ba9f", "metadata": {}, "outputs": [ { "name": "stdin", "output_type": "stream", "text": [ "Paste API key here: ········\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Na-26-001\n" ] } ], "source": [ "resources_ids = [ i for i in sample_dict ]\n", "first_sample = resources_ids[0]\n", "\n", "apikey = getpass(\"Paste API key here: \")\n", "sample_data = call_sample(apikey, first_sample)\n", "sample_title = sample_data[\"title\"]\n", "print(sample_title)" ] }, { "cell_type": "markdown", "id": "ba4f0459-8da0-494d-b0c2-23dae509538c", "metadata": {}, "source": [ "Now all that's left for us to do is merge the results to create a single dictionary with the name of the sample and its different layers." ] }, { "cell_type": "markdown", "id": "d714bde9-73a2-4365-b54e-d129533aa3de", "metadata": {}, "source": [ "## Basic parser\n", "The parser needs:\n", "* The code from the section \"*Multiple layers from uncategorized files*\" responsible for fetching and grouping data on the layers.\n", "* The `find_missing` and `find_duplicates` functions.\n", "* The code from the previous section to collect the names of the samples." ] }, { "cell_type": "code", "execution_count": 29, "id": "daa59593-fd40-4b8a-b7f5-5cdbd6482fc3", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdin", "output_type": "stream", "text": [ "Paste API key here: ········\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"Na-26-001\": {\n", " \"instrument\": {\n", " \"deposition_chamber\": 72,\n", " \"laser_system\": \"Excimer \",\n", " \"rheed_system\": \"staib\"\n", " },\n", " \"multilayer\": {\n", " \"layer_1\": {\n", " \"operator\": \"Emiliano Di Gennaro\",\n", " \"created_at\": \"2026-01-20 16:11:32\",\n", " \"sample\": {\n", " \"type\": \"items\",\n", " \"value\": 855,\n", " \"group_id\": 4,\n", " \"position\": 0\n", " },\n", " \"temperature\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"760\",\n", " \"group_id\": 4,\n", " \"position\": 7\n", " },\n", " \"target\": {\n", " \"type\": \"items\",\n", " \"value\": 854,\n", " \"group_id\": 4,\n", " \"position\": 2,\n", " \"required\": true\n", " }\n", " },\n", " \"layer_2\": {\n", " \"operator\": \"Emiliano Di Gennaro\",\n", " \"created_at\": \"2026-01-20 16:18:48\",\n", " \"sample\": {\n", " \"type\": \"items\",\n", " \"value\": 855,\n", " \"group_id\": 4,\n", " \"position\": 0\n", " },\n", " \"temperature\": {\n", " \"type\": \"number\",\n", " \"unit\": \"\\u00b0C\",\n", " \"units\": [\n", " \"\\u00b0C\"\n", " ],\n", " \"value\": \"500\",\n", " \"group_id\": 4,\n", " \"position\": 7\n", " },\n", " \"target\": {\n", " \"type\": \"items\",\n", " \"value\": 466,\n", " \"group_id\": 4,\n", " \"position\": 2,\n", " \"required\": true\n", " }\n", " }\n", " }\n", " }\n", "}\n" ] } ], "source": [ "import os, json, requests\n", "from getpass import getpass\n", "\n", "def valid_elabfiles(path):\n", " '''Lookup directory \"path\" and\n", " returns list of valid eLabFTW\n", " Experiment JSON files.'''\n", " elabfiles = []\n", " for filename in os.listdir(path):\n", " if filename.endswith(\".json\"):\n", " try:\n", " with open(os.path.join(path, filename), \"r\") as f:\n", " data = json.load(f)\n", " if data.get(\"elabid\"): # insert specific NeXus requirements here later\n", " \n", " elabfiles.append(filename)\n", " f.close()\n", " except json.decoder.JSONDecodeError as e: # invalid files \"masked\" as JSON\n", " #print(f\"wait a moment: {e}\") # just for debug\n", " pass\n", " return elabfiles\n", "\n", "def call_sample(apikey, elabid, SERVER_URL=\"https://elabftw.fisica.unina.it/\"): # TO-DO: rm default server\n", " '''Queries the Resources (/items) API endpoint\n", " of eLabFTW instance to request data (JSON)\n", " on a certain sample given its eLab-ID.\n", " \n", " Requires an active (RO/RW) API key.\n", " Defaults to elabftw.fisica.unina.it.'''\n", " full_elab_url = f\"{SERVER_URL}api/v2\" # API endpoint root for eLabFTW\n", " items_url = f\"{full_elab_url}/items\" # API endpoint /items\n", " header = {\n", " \"Authorization\": apikey,\n", " \"Content-Type\": \"application/json\"\n", " }\n", " sample = requests.get(\n", " headers=header,\n", " url=f\"{items_url}/{elabid}\",\n", " verify=True\n", " )\n", " return sample.json()\n", "\n", "def id2sample(apikey, elabid):\n", " '''Fetches sample data (JSON) from eLabFTW\n", " instance (using function \"call_sample()\")\n", " and extracts significant information.\n", " \n", " Currently, it only returns the sample's title.'''\n", " #apikey = getpass(\"Paste API key here: \") # move outside loops\n", " sample_data = call_sample(apikey, elabid)\n", " sample_title = sample_data[\"title\"]\n", " return sample_title\n", "\n", "def fetch_and_group(path):\n", " '''Fetches experiment data from eLabFTW JSON\n", " files in a given folder, then \n", " '''\n", " sample_dict = {}\n", " apikey = getpass(\"Paste API key here: \")\n", " for filename in valid_elabfiles(path):\n", " with open(os.path.join(path, filename), \"r\") as f:\n", " layer = json.load(f)\n", " extra = layer[\"metadata_decoded\"][\"extra_fields\"]\n", " sample_id = extra[\"Sample\"][\"value\"]\n", " sample_title = id2sample(apikey, sample_id)\n", " lpn = int(extra[\"Layer Progressive Number\"][\"value\"]) # Layer Progressive Number\n", " if not sample_dict.get(sample_title): # if not existent yet, initialize\n", " sample_dict[sample_title] = {\n", " \"instrument\": {\n", " \"deposition_chamber\": extra[\"Chamber\"][\"value\"], # ID of associated resource (PLD chamber) - useless as is!\n", " \"laser_system\": extra[\"Laser System\"][\"value\"],\n", " \"rheed_system\": extra[\"RHEED System\"][\"value\"]\n", " },\n", " \"multilayer\": {}\n", " }\n", " sample_dict[sample_title][\"multilayer\"][f\"layer_{lpn}\"] = {\n", " \"operator\": layer[\"fullname\"],\n", " \"created_at\": layer[\"created_at\"],\n", " \"sample\": extra[\"Sample\"], # ID of associated sample - useless as is!\n", " \"temperature\": extra[\"Heater temperature \"], # space at the end is a config error in eLab!\n", " \"target\": extra[\"Target\"]\n", " }\n", " return sample_dict\n", "\n", "\n", "sample_dict = fetch_and_group(\"../tests/objects\")\n", "print(json.dumps(sample_dict, indent=3))" ] }, { "cell_type": "markdown", "id": "de1b1870-7fc3-4ee5-8cce-c098e5bf909a", "metadata": {}, "source": [ "For debug purposes, let's see which info is included in the sample_dict dictionary." ] }, { "cell_type": "code", "execution_count": 52, "id": "0fc6e88f-881d-413a-bfe1-377213f7dda2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "# Info about sample Na-26-001:\n", "* The deposition chamber is 72.\n", "* The laser system is EXCIMER.\n", "* The RHEED system is STAIB.\n", "\n", "## Layers of Na-26-001:\n", "\n", "### layer_1\n", "* It was created at 2026-01-20 16:11:32.\n", "* The operator was Emiliano Di Gennaro.\n", "* The deposition temperature was 760 °C.\n", "* The target eLabID was 854.\n", "\n", "### layer_2\n", "* It was created at 2026-01-20 16:18:48.\n", "* The operator was Emiliano Di Gennaro.\n", "* The deposition temperature was 500 °C.\n", "* The target eLabID was 466.\n" ] } ], "source": [ "for sample in sample_dict:\n", " print(f\"# Info about sample {sample}:\")\n", " multilayer = sample_dict[sample][\"multilayer\"]\n", " instrument = sample_dict[sample][\"instrument\"]\n", " deposition_chamber = instrument[\"deposition_chamber\"] # integer\n", " laser_system = str(instrument[\"laser_system\"]).strip().upper() # string\n", " rheed_system = str(instrument[\"rheed_system\"]).strip().upper() # string\n", " \n", " print(f\"* The deposition chamber is {deposition_chamber}.\")\n", " print(f\"* The laser system is {laser_system}.\")\n", " print(f\"* The RHEED system is {rheed_system}.\")\n", " print(f\"\\n## Layers of {sample}:\")\n", " for layer in multilayer:\n", " print(f\"\\n### {layer}\")\n", " layerdata = multilayer[layer]\n", " operator = layerdata[\"operator\"]\n", " created_at = layerdata[\"created_at\"]\n", " temperature = layerdata[\"temperature\"][\"value\"]\n", " temperature_unit = layerdata[\"temperature\"][\"unit\"]\n", " target = layerdata[\"target\"][\"value\"]\n", " \n", " print(f\"* It was created at {created_at}.\")\n", " print(f\"* The operator was {operator}.\")\n", " print(f\"* The deposition temperature was {temperature} {temperature_unit}.\")\n", " print(f\"* The target eLabID was {target}.\")" ] }, { "cell_type": "markdown", "id": "e16c6f3d-cc3a-45c1-9988-bbf0de3baf08", "metadata": {}, "source": [ "## To the next level: creating a dictionary with the same hierarchy as the final NeXus file\n", "\n", "```\n", "pld_fabrication\n", "|-sample\n", "| |-substrate\n", "| | |-name\n", "| |-multilayer\n", "| | |-LAYER\n", "| | | |-target\n", "| | | | |-name\n", "| | | | |-chemical_formula\n", "\n", "```" ] }, { "cell_type": "code", "execution_count": null, "id": "a615c496-8cb9-451f-a088-beb4672379bf", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.3" } }, "nbformat": 4, "nbformat_minor": 5 }