update pyproject

.gitignore

@@ -5,3 +5,5 @@
 /data
 backend/api_logs.txt
 *__pycache__*
+.env
+poetry.lock

backend/api/pipelines.py

@@ -0,0 +1,215 @@
+def parse_int(args):
+    try:
+       return int( args )
+    except ValueError:
+        print(f"{args} cannot be cast to int")
+        raise
+
+def mangrove_by_country_latest():
+    return """
+           select * from models_final.final__protected_mangroves_summary_stats_by_country_agg
+           where year = '2020'
+           order by cumulative_pixels_diff desc
+           """
+
+def bitcoin_business_growth_timeseries(args):
+    days_ago = parse_int(args["days_ago"])
+    country_name = args["country_name"]
+    return f"""
+           select * from models_final.final__bitcoin_business_growth_by_country
+           where days_ago <= {days_ago} and country_name = '{country_name}'
+           order by date
+           """
+
+def bitcoin_business_growth_percent_diff_days_ago(args):
+    days_ago = parse_int(args["days_ago"])
+    return f"""
+           with
+               filtered_data as (
+                   select country_name, date, days_ago, cumulative_value
+                   from models_final.final__bitcoin_business_growth_by_country
+                   where days_ago <= {days_ago}
+                   order by country_name, days_ago desc
+               ),
+               first_and_last_values as (
+                   select
+                       country_name,
+                       date,
+                       days_ago,
+                       cumulative_value,
+                       first_value(cumulative_value) over (
+                           partition by country_name order by days_ago desc
+                       ) as first_value,
+                       first_value(date) over (
+                           partition by country_name order by days_ago desc
+                       ) as first_date,
+                       first_value(cumulative_value) over (
+                           partition by country_name order by days_ago
+                       ) as last_value,
+                       first_value(date) over (
+                           partition by country_name order by days_ago
+                       ) as last_date
+                   from filtered_data
+               ),
+               diff as (
+                   select
+                       country_name,
+                       date,
+                       first_date,
+                       last_date,
+                       days_ago,
+                       cumulative_value,
+                       first_value,
+                       last_value,
+                       last_value - first_value as difference,
+                       round(
+                           100 * safe_divide((last_value - first_value), first_value), 2
+                       ) as percent_difference
+                   from first_and_last_values
+               )
+           select *
+           from diff
+           where days_ago = 1
+           order by difference desc
+           """
+# def bitcoin_business_growth_timeseries(query):
+#     pipeline = [
+#         {
+#             "$match": {
+#                 "days_ago": {"$lte": int(query["days_ago"])},
+#                 "country_name": query["country_name"],
+#             }
+#         },
+#         {
+#             "$project": {
+#                 "country_name": "$country_name",
+#                 "date": "$date",
+#                 "cumulative_value": "$cumulative_value",
+#             }
+#         },
+#         {"$sort": {"country_name": 1, "days_ago": 1}},
+#     ]
+#     return pipeline
+# def mangrove_by_country_latest():
+#     pipeline = [
+#         {
+#             "$match": {"year": "2020"},
+#         },
+#     ]
+#     return pipeline
+#
+#
+# def mangrove_by_country_agg(query):
+#     pipeline = [
+#         {"$match": {"country_with_parent": query["country_with_parent"]}},
+#         {
+#             "$group": {
+#                 "_id": {"country_with_parent": "$country_with_parent", "year": "$year"},
+#                 "total_pixels": {"$sum": "$total_n_pixels"},
+#             }
+#         },
+#         {
+#             "$project": {
+#                 "_id": 0,
+#                 "country_with_parent": "$_id.country_with_parent",
+#                 "year": "$_id.year",
+#                 "total_pixels": 1,
+#             }
+#         },
+#         {"$sort": {"year": 1}},
+#     ]
+#     return pipeline
+#
+#
+# def bitcoin_business_growth_timeseries(query):
+#     pipeline = [
+#         {
+#             "$match": {
+#                 "days_ago": {"$lte": int(query["days_ago"])},
+#                 "country_name": query["country_name"],
+#             }
+#         },
+#         {
+#             "$project": {
+#                 "country_name": "$country_name",
+#                 "date": "$date",
+#                 "cumulative_value": "$cumulative_value",
+#             }
+#         },
+#         {"$sort": {"country_name": 1, "days_ago": 1}},
+#     ]
+#     return pipeline
+#
+#
+# def bitcoin_business_growth_percent_diff_days_ago(query):
+    pipeline = [
+        {"$match": {"days_ago": {"$lte": int(query["days_ago"])}}},
+        {"$sort": {"country_name": 1, "days_ago": 1}},
+        {
+            "$group": {
+                "_id": "$country_name",
+                "firstvalue": {"$first": "$cumulative_value"},
+                "lastvalue": {"$last": "$cumulative_value"},
+                "firstdate": {"$min": "$date"},
+                "lastdate": {"$max": "$date"},
+            }
+        },
+        {
+            "$project": {
+                "country_name": "$_id",
+                "first_value": "$firstvalue",
+                "last_value": "$lastvalue",
+                "difference": {
+                    "$subtract": [
+                        {"$todouble": "$firstvalue"},
+                        {"$todouble": "$lastvalue"},
+                    ]
+                },
+                "first_date": "$firstdate",
+                "last_date": "$lastdate",
+                "percent_difference": {
+                    "$cond": {
+                        "if": {"$eq": [{"$todouble": "$lastvalue"}, 0]},
+                        "then": {
+                            "$cond": {
+                                "if": {"$gt": [{"$todouble": "$firstvalue"}, 0]},
+                                "then": "new",
+                                "else": "none",
+                            }
+                        },
+                        "else": {
+                            "$round": [
+                                {
+                                    "$multiply": [
+                                        {
+                                            "$divide": [
+                                                {
+                                                    "$subtract": [
+                                                        {"$todouble": "$firstvalue"},
+                                                        {"$todouble": "$lastvalue"},
+                                                    ]
+                                                },
+                                                {"$todouble": "$lastvalue"},
+                                            ]
+                                        },
+                                        100,
+                                    ]
+                                }
+                            ]
+                        },
+                    }
+                },
+            }
+        },
+    ]
+    return pipeline
+#
+#
+# def bitcoin_business_growth_latest(query):
+#     pipeline = [
+#         {
+#             "$match": query["filter"],
+#         },
+#         {"$sort": {"date": 1}},
+#     ]
+#     return pipeline

backend/api/postgres_handler.py

@@ -0,0 +1,34 @@
+from psycopg2.extras import RealDictCursor
+import psycopg2, os
+
+class PostgresHandler:
+    def __init__(self):
+        self.connection = self.connect_to_pg()
+        self.cur = self.connection.cursor(cursor_factory=RealDictCursor)
+
+    def connect_to_pg(self):
+        try:
+            connection = psycopg2.connect(
+                dbname=os.getenv('PGDATABASE'),
+                host=os.getenv('PGHOST'),
+                user=os.getenv('PGUSER'),
+                password=os.getenv('PGPASSWORD'),
+                port=os.getenv('PGPORT'),
+            )
+        except Exception as e:
+            message=f"Connection to postgres database failed: {e}"
+            raise Exception(message)
+        print(f"Successfully connected to DB")
+        return connection
+
+    def execute_query(self, query):
+        try:
+            self.cur.execute(query)
+            results = self.cur.fetchall()
+            self.connection.commit()
+            self.connection.close()
+            return results
+        except Exception:
+            print("Error executing query")
+            raise
+

backend/api/route.py

@@ -0,0 +1,96 @@
+from fastapi import APIRouter
+from api.postgres_handler import PostgresHandler
+import api.pipelines as pipelines
+import api.schemas as schemas
+from api.schemas import DataSerializer
+import json
+
+router = APIRouter()
+
+def parse_args_to_dict(query):
+    try:
+        return json.loads(query)
+    except json.JSONDecodeError as e:
+        return {"error": f"Invalid JSON: {e}"}
+
+@router.get("/mangrove_by_country_latest")
+async def mangrove_by_country_latest():
+    pipeline = pipelines.mangrove_by_country_latest()
+    handler = PostgresHandler()
+
+    schema = schemas.mangrove_by_country_latest_schema
+    serializer = DataSerializer(schema)
+    rawData = handler.execute_query(pipeline)
+    serializedData = serializer.serialize_many(rawData)
+    return serializedData
+
+@router.get("/bitcoin_business_growth_timeseries")
+async def bitcoin_business_growth_timeseries(query: str):
+    args = parse_args_to_dict(query)
+
+    pipeline = pipelines.bitcoin_business_growth_timeseries(args)
+    handler = PostgresHandler()
+
+    schema = schemas.bitcoin_business_growth_timeseries_schema
+    serializer = DataSerializer(schema)
+
+    rawData = handler.execute_query(pipeline)
+    serializedData = serializer.serialize_many(rawData)
+
+    return serializedData
+
+@router.get("/bitcoin_business_growth_percent_diff")
+async def bitcoin_business_growth_percent_diff(query: str):
+    args = parse_args_to_dict(query)
+
+    pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(args)
+    handler = PostgresHandler()
+
+    schema = schemas.bitcoin_business_growth_percent_diff_schema
+    serializer = DataSerializer(schema)
+
+    rawData = handler.execute_query(pipeline)
+    serializedData = serializer.serialize_many(rawData)
+    return serializedData
+
+# @router.get("/bitcoin_business_growth_percent_diff")
+# async def bitcoin_business_growth_percent_diff(query: str):
+#     query = ast.literal_eval(query)
+#     
+#     query = queries.bitcoin_business_growth_percent_diff_days_ago(query)
+#     handler = PostgresHandler(connection)
+#
+#     schema = schemas.bitcoin_business_growth_percent_diff_schema
+#     pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(query)
+#     serializer = DataSerializer(schema)
+#     handler = MongoDBHandler(collection_name)
+#     rawData = handler.aggregate(pipeline)
+#     serializedData = serializer.serialize_many(rawData)
+#     return serializedData
+# @router.get("/mangrove_by_country_agg")
+# async def mangrove_by_country_agg(query: str):
+#     query = ast.literal_eval(query)
+#     db = client.baseddata
+#     collection_name = db["final__protected_mangroves_summary_stats_by_country_agg"]
+#     schema = schemas.mangrove_by_country_agg_schema
+#     pipeline = pipelines.mangrove_by_country_agg(query)
+#     serializer = DataSerializer(schema)
+#     handler = MongoDBHandler(collection_name)
+#     rawData = handler.aggregate(pipeline)
+#     serializedData = serializer.serialize_many(rawData)
+#     return serializedData
+#
+
+# @router.get("/bitcoin_business_growth_timeseries")
+# async def bitcoin_business_growth_timeseries(query: str):
+#     query = ast.literal_eval(query)
+#     db = client.baseddata
+#     collection_name = db["final__bitcoin_business_growth_by_country"]
+#     schema = schemas.bitcoin_business_growth_timeseries_schema
+#     pipeline = pipelines.bitcoin_business_growth_timeseries(query)
+#     serializer = DataSerializer(schema)
+#     handler = MongoDBHandler(collection_name)
+#     rawData = handler.aggregate(pipeline)
+#     serializedData = serializer.serialize_many(rawData)
+#     return serializedData
+

backend/api/schemas.py

@@ -0,0 +1,42 @@
+def mangrove_by_country_latest_schema(data):
+    return {
+        "country_with_parent": str(data["country_with_parent"]),
+        "original_pixels": int(data["original_pixels"]),
+        "total_n_pixels": int(data["total_n_pixels"]),
+        "cumulative_pixels_diff": int(data["cumulative_pixels_diff"]),
+        "cumulative_pct_diff": float(data["cumulative_pct_diff"]),
+    }
+
+def mangrove_by_country_agg_schema(data):
+    return {
+        "country_with_parent": str(data["country_with_parent"]),
+        "year": int(data["year"]),
+        "total_pixels": int(data["total_pixels"])
+    }
+
+def bitcoin_business_growth_percent_diff_schema(data):
+    return {
+        "country_name": str(data["country_name"]),
+        "date_range": str(f'{data["first_date"]} to {data["last_date"]}'),
+        "first_value": int(data["first_value"]),
+        "last_value": int(data["last_value"]),
+        "difference": int(data["difference"]),
+        "percent_difference": str(data["percent_difference"])
+    }
+
+def bitcoin_business_growth_timeseries_schema(data):
+    return {
+        "country_name": str(data["country_name"]),
+        "date": data["date"],
+        "cumulative_value": int(data["cumulative_value"])
+    }
+
+class DataSerializer:
+    def __init__(self, schema_func):
+        self.schema_func = schema_func
+
+    def serialize_one(self, data) -> dict:
+        return self.schema_func(dict( data ))
+
+    def serialize_many(self, data_list) -> list:
+        return [self.serialize_one(data) for data in data_list]

backend/app.py

@@ -1,210 +0,0 @@
-from flask import Flask, g, jsonify, request, json, Response, send_from_directory, abort
-from flask_cors import CORS
-import orjson, os
-import pandas as pd
-
-import datetime
-import time
-
-app = Flask(__name__)
-CORS(app)
-
-FILES_DIRECTORY = "../data/"
-
-
-@app.before_request
-def start_timer():
-    g.start = time.time()
-
-
-@app.after_request
-def log(response):
-    now = time.time()
-    duration = round(now - g.start, 4)
-    dt = datetime.datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
-
-    log_entry = {
-        "timestamp": dt,
-        "duration": duration,
-        "method": request.method,
-        "url": request.url,
-        "status": response.status_code,
-        "remote_addr": request.access_route[-1],
-        "user_agent": request.user_agent.string,
-    }
-
-    log_line = ",".join(f"{key}={value}" for key, value in log_entry.items())
-
-    with open("api_logs.txt", "a") as f:
-        f.write(log_line + "\n")
-
-    return response
-
-
-@app.route("/bitcoin_business_growth_by_country", methods=["GET"])
-def business_growth():
-
-    today = datetime.datetime.today()
-
-    # Parse args from request
-    latest_date = request.args.get("latest_date")
-    country_names = request.args.get("countries")
-    cumulative_period_type = request.args.get("cumulative_period_type")
-
-    # Open json locally
-    with open("../data/final__bitcoin_business_growth_by_country.json", "rb") as f:
-        data = orjson.loads(f.read())
-
-    # Filter based on args
-    if latest_date:
-        latest_date_bool = latest_date == "true"
-        filtered_data = [
-            item for item in data if item["latest_date"] == latest_date_bool
-        ]
-    else:
-        filtered_data = data
-
-    if country_names:
-        countries = [name.strip() for name in country_names.split(",")]
-        filtered_data = [
-            item for item in filtered_data if item["country_name"] in countries
-        ]
-
-    if cumulative_period_type == "1 day":
-        delta = today - datetime.timedelta(days=2)
-        filtered_data = [
-            item
-            for item in filtered_data
-            if item["cumulative_period_type"] == cumulative_period_type
-            and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
-        ]
-    elif cumulative_period_type == "7 day":
-        delta = today - datetime.timedelta(days=8)
-        filtered_data = [
-            item
-            for item in filtered_data
-            if item["cumulative_period_type"] == cumulative_period_type
-            and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
-        ]
-    elif cumulative_period_type == "28 day":
-        delta = today - datetime.timedelta(days=29)
-        filtered_data = [
-            item
-            for item in filtered_data
-            if item["cumulative_period_type"] == cumulative_period_type
-            and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
-        ]
-    elif cumulative_period_type == "365 day":
-        delta = today - datetime.timedelta(days=366)
-        filtered_data = [
-            item
-            for item in filtered_data
-            if item["cumulative_period_type"] == cumulative_period_type
-            and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
-        ]
-
-    # Sort by date
-    sorted_data = sorted(filtered_data, key=lambda x: x["date"], reverse=False)
-
-    # Return json
-    return Response(json.dumps(sorted_data), mimetype="application/json")
-
-
-@app.route("/get_json/<filename>", methods=["GET"])
-def get_json(filename):
-
-    period = request.args.get("period")
-    today = datetime.datetime.today()
-
-    file_path = os.path.join(FILES_DIRECTORY, filename)
-    if not os.path.isfile(file_path):
-        abort(404)
-
-    with open(file_path, "r") as f:
-        data = orjson.loads(f.read())
-
-    if period == "last 7 days":
-        delta = today - datetime.timedelta(days=7)
-        filtered_data = [
-            item
-            for item in data
-            if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
-        ]
-        sorted_data = sorted(filtered_data, key=lambda x: x["date"])
-    elif period == "last 28 days":
-        delta = today - datetime.timedelta(days=28)
-        filtered_data = [
-            item
-            for item in data
-            if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
-        ]
-        sorted_data = sorted(filtered_data, key=lambda x: x["date"])
-    elif period == "last 365 days":
-        delta = today - datetime.timedelta(days=365)
-        filtered_data = [
-            item
-            for item in data
-            if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
-        ]
-        sorted_data = sorted(filtered_data, key=lambda x: x["date"])
-    elif period == "last 2 years":
-        delta = today - datetime.timedelta(days=730)
-        filtered_data = [
-            item
-            for item in data
-            if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
-        ]
-        sorted_data = sorted(filtered_data, key=lambda x: x["date"])
-    else:
-        sorted_data = sorted(data, key=lambda x: x["date"])
-
-    return jsonify(sorted_data)
-
-
-@app.route("/mangrove_data/<method>", methods=["GET"])
-def mangrove_data(method):
-
-    with open("../data/dev/final__wdpa_pid_mangrove_diff_stats.json", "rb") as f:
-        data = orjson.loads(f.read())
-
-    if method == "countries":
-        df = pd.read_json(json.dumps(data))
-        countries = df[["year", "country", "n_pixels", "diff", "cumulative_diff"]]
-        countriesAgg = countries.groupby(["year", "country"]).agg(
-            {"n_pixels": "sum", "diff": "sum", "cumulative_diff": "sum"}
-        )
-        countriesAgg["year0_pixels"] = (
-            countriesAgg["n_pixels"] - countriesAgg["cumulative_diff"]
-        )
-        countriesAgg["pct_diff"] = (
-            100
-            * (countriesAgg["n_pixels"] - countriesAgg["year0_pixels"])
-            / countriesAgg["year0_pixels"]
-        ).round(2)
-        countriesLatest = countriesAgg.loc[[2020]].reset_index().set_index("country")
-        return Response(
-            countriesLatest.to_json(orient="index"), mimetype="application/json"
-        )
-
-
-@app.route("/download/<filename>", methods=["GET"])
-def download_file(filename):
-    try:
-        return send_from_directory(FILES_DIRECTORY, filename, as_attachment=True)
-    except FileNotFoundError:
-        abort(404)
-
-
-@app.route("/cog", methods=["GET"])
-def serve_cog():
-    year = request.args.get("year")
-    pid = request.args.get("pid")  # change this line
-    dir = f"{FILES_DIRECTORY}/cog/{year}/"
-    try:
-        return send_from_directory(dir, f"{pid}.tif", as_attachment=True)
-    except FileNotFoundError:
-        abort(404)
-
-
-if __name__ == "__main__":
-    app.run()

backend/main.py

@@ -0,0 +1,6 @@
+from fastapi import FastAPI
+from api.route import router
+
+app = FastAPI()
+
+app.include_router(router)

content/data-lab/global-business-growth.md

@@ -15,8 +15,10 @@ You can select the growth period of interest from the drop-down, which updates t
 The chart always reflects the countries selected in the table.
 
 <br/>
-{{< chart src="/js/bitcoin-business-growth-chart.js" >}}
-{{< table src="/js/bitcoin-business-growth-table.js" >}}
+{{< dropdown_filter id="days_ago_dropdown_filter" id_filter="days_ago" options="1 day:1,7 day:7,28 day:28,1 year:365,5 year:1826,10 year:3652,all time:10000" default_selection="7 day" targets="bitcoin-business-growth-chart bitcoin-business-growth-table" >}}
+{{< table id="bitcoin-business-growth-table" endpoint="bitcoin_business_growth_percent_diff" headers="{'country_name': 'Country', 'date_range': 'Date Range', 'first_value': 'Previous #', 'last_value': 'Current #', 'difference': 'Diff', 'percent_difference': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_name" selectableRows="multi" targets="bitcoin-business-growth-chart" defaultFirstSelected="true" >}}
+
+{{< chart id="bitcoin-business-growth-chart" endpoint="bitcoin_business_growth_timeseries" chartType="line" xAxisField="date" yAxisField="cumulative_value" scaleChart=true >}}
 
 #### Attribution and License
 Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright)

content/data-lab/protected-mangroves.md

@@ -9,5 +9,8 @@ tags: ["Bitcoin", "Stats"]
 script: "/js/mangrove-map.js"
 ---
 
-{{< table id="mangrove_countries" endpoint="mangrove_data/countries" tableKey="country" headers="{'country': 'Country', 'cumulative_diff': 'Diff', 'pct_diff': '% Diff'}" maxHeight="400px" sortable="true">}}
+{{< table id="mangrove_countries" endpoint="mangrove_by_country_latest" headers="{'country_with_parent': 'Country', 'original_pixels': '1996 Cover', 'total_n_pixels': '2020 Cover', 'cumulative_pixels_diff': 'Diff', 'cumulative_pct_diff': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_with_parent" selectableRows="single" defaultFirstSelected="true" >}}
+{{< chart id="mangrove_countries" endpoint="mangrove_by_country_agg" chartType="bar" xAxisField="year" yAxisField="total_pixels" scaleChart=true >}}
 {{< map id="map" style="https://tiles.semitamaps.com/styles/maptiler-basic/style.json">}}
+
+{{< chart id="mangrove-country-timeseries-chart" endpoint="mangrove_country_timeseries" chartType="line" xAxisField="date" yAxisField="n_pixels" scaleChart=true >}}

hugo.toml

@@ -3,7 +3,7 @@ languageCode = 'en-gb'
 title = 'Based Data'
 
 [params]
-    apiURL = 'http://localhost:5000'
+    apiURL = 'http://localhost:8000'
 
 [markup.highlight]
     pygmentsUseClasses = false

layouts/partials/chart.html

@@ -1,8 +1,96 @@
 <script src="https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js"></script>
-<section class="chart-container">
-  <div id="chart-modifiers"></div>
-  <div id="chart">
-    <script src="{{ .src }}"></script>
-  </div>
-</section>
+<script>
+  chartData = [];
+  function createChart(
+    id,
+    endpoint,
+    chartType,
+    xAxisField,
+    yAxisField,
+    sortField = null,
+    scaleChart = false,
+  ) {
+    async function fetchDataForChart(query, valueId) {
+      try {
+        const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
+        const response = await fetch(apiEndpoint);
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const fetchedData = await response.json();
+        const newData = fetchedData.reduce((acc, item) => {
+          const objectId = item[valueId];
+          if (!acc[objectId]) {
+            acc[objectId] = [];
+          }
+          acc[objectId].push([item[xAxisField], item[yAxisField]]);
+          return acc;
+        }, {});
+        chartData = { ...chartData, ...newData };
+        updateChart();
+      } catch (error) {
+        console.error("Fetching data failed:", error);
+      }
+    }
+
+    function updateChart() {
+      console.log(chartData);
+      let chartDataMap = new Map();
+      for (let objectId in chartData) {
+        chartDataMap.set(objectId, chartData[objectId]);
+      }
+      var chartDom = document.getElementById(`${id}`);
+      var myChart = echarts.init(chartDom);
+
+      var option = {
+        tooltip: {
+          ...tooltip,
+          valueFormatter(value, index) {
+            return nFormatter(value, 0);
+          },
+        },
+        xAxis: {
+          type: "time",
+        },
+        yAxis: {
+          scale: scaleChart,
+          type: "value",
+        },
+        series: Array.from(chartDataMap.entries()).map(([name, data]) => ({
+          name,
+          type: chartType,
+          data,
+          showSymbol: false,
+        })),
+      };
+
+      myChart.setOption(option, true);
+    }
+
+    // listen for filter events for this target
+    document.addEventListener("filterChange", function (event) {
+      tableId = document.getElementById(id).id;
+      console.log(event.detail);
+      eventDetail = event.detail;
+      if (eventDetail.filterActions.includes("refresh")) {
+        chartData = [];
+        updateChart();
+      } else {
+        if (eventDetail.filterTargets.includes(tableId)) {
+          if (eventDetail.filterActions.includes("selected")) {
+            valueId = eventDetail.filterId;
+            let selectedRow = {
+              [valueId]: eventDetail.filterValue,
+            };
+            query = queryConstructor(selectedRow);
+            fetchDataForChart(query, valueId);
+          } else {
+            delete chartData[eventDetail.filterValue];
+            updateChart();
+          }
+        }
+      }
+    });
+  }
+</script>
 <script src="/js/chart-params.js"></script>

layouts/partials/head.html

@@ -8,6 +8,8 @@
   <link rel="stylesheet" href="/css/toc.css" type="text/css" media="all" />
   <link rel="stylesheet" href="/css/articles.css" type="text/css" media="all" />
   <link rel="stylesheet" href="/css/charts.css" type="text/css" media="all" />
+  <script src="/js/lib/sorttable.js"></script>
+  <script src="/js/lib/helper-functions.js"></script>
   <link
     rel="stylesheet"
     href="/css/codeblock.css"

layouts/partials/table.html

@@ -0,0 +1,142 @@
+<script>
+  function createTable(
+    endpoint,
+    id,
+    headers,
+    maxHeight,
+    sortable,
+    valueId,
+    selectableRows,
+    filterTargets,
+    defaultFirstSelected,
+  ) {
+    async function fetchDataForTable(query) {
+      try {
+        const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
+        const response = await fetch(apiEndpoint);
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const fetchedData = await response.json();
+        data = fetchedData;
+        generateTable(data);
+      } catch (error) {
+        console.error("Fetching data failed:", error);
+      }
+    }
+
+    function generateTable(data) {
+      const jsonTableContainer = document.getElementById(`${id}--container`);
+      jsonTableContainer.className = "jsonTableContainer";
+      jsonTableContainer.innerHTML = "";
+      jsonTableContainer.style.maxHeight = maxHeight;
+
+      tableHeaderNames = Object.values(headers);
+      tableHeaderKeys = Object.keys(headers);
+
+      const table = document.createElement("table");
+      table.id = `${id}`;
+      const thead = document.createElement("thead");
+      const tbody = document.createElement("tbody");
+      const headerRow = document.createElement("tr");
+      tableHeaderNames.forEach((header) => {
+        const th = document.createElement("th");
+        th.textContent = header;
+        headerRow.appendChild(th);
+      });
+      thead.appendChild(headerRow);
+      table.appendChild(thead);
+
+      for (const key in data) {
+        const row = document.createElement("tr");
+        row.value = data[key][valueId];
+        tableHeaderKeys.forEach((columnName) => {
+          const td = document.createElement("td");
+          const div = document.createElement("div");
+          div.id = "scrollable";
+          div.textContent = data[key][columnName];
+          td.appendChild(div);
+          row.appendChild(td);
+          tbody.appendChild(row);
+        });
+      }
+      table.appendChild(thead);
+      table.appendChild(tbody);
+      jsonTableContainer.appendChild(table);
+
+      // sortable
+      if (sortable == "true") {
+        table.className = "sortable";
+        sorttable.makeSortable(document.getElementById(`${id}`));
+      }
+
+      if (selectableRows === "multi" || selectableRows === "single") {
+        const rows = table.getElementsByTagName("tr");
+        for (let i = 1; i < rows.length; i++) {
+          rows[i].addEventListener("click", function () {
+            if (selectableRows === "multi") {
+              this.classList.toggle("selected");
+              if (this.classList.contains("selected")) {
+                const event = new CustomEvent("filterChange", {
+                  detail: {
+                    filterId: valueId,
+                    filterValue: this.value,
+                    filterActions: ["selected"],
+                    filterTargets: filterTargets,
+                  },
+                });
+                document.dispatchEvent(event);
+              } else {
+                const event = new CustomEvent("filterChange", {
+                  detail: {
+                    filterId: valueId,
+                    filterValue: this.value,
+                    filterActions: ["deselected"],
+                    filterTargets: filterTargets,
+                  },
+                });
+                document.dispatchEvent(event);
+              }
+            } else if (selectableRows === "single") {
+              if (this.classList.contains("selected")) {
+                this.classList.remove("selected");
+              } else {
+                for (let j = 1; j < rows.length; j++) {
+                  rows[j].classList.remove("selected");
+                }
+                this.classList.add("selected");
+              }
+            }
+          });
+          if (defaultFirstSelected == true) {
+            if (i == 1) {
+              rows[i].classList.add("selected");
+              const event = new CustomEvent("filterChange", {
+                detail: {
+                  filterId: valueId,
+                  filterValue: rows[i].value,
+                  filterActions: ["selected"],
+                  filterTargets: filterTargets,
+                },
+              });
+              document.dispatchEvent(event);
+            }
+          }
+        }
+      }
+
+    }
+
+    // listen for filter events for this target
+    document.addEventListener("filterChange", function (event) {
+      tableId = document.getElementById(id).id;
+      if (event.detail.filterTargets.includes(tableId)) {
+        query = queryConstructor();
+        fetchDataForTable(query);
+      }
+    });
+
+    query = queryConstructor();
+    fetchDataForTable(query);
+  }
+</script>

layouts/shortcodes/chart.html

@@ -1,2 +1,10 @@
-{{ $id := .Get "src" | md5 }} {{ partial "chart.html" (dict "src" (.Get "src")
-"id" $id) }}
+{{ partial "chart.html" }}
+<section class = 'chart-container'>
+<div class = "chart" id='{{ .Get "id" }}'>
+<script>
+document.addEventListener("DOMContentLoaded", function () {
+  createChart(id={{ .Get "id" }}, endpoint={{ .Get "endpoint" }}, chartType={{ .Get "chartType" }},  xAxisField={{ .Get "xAxisField" }}, yAxisField={{ .Get "yAxisField" }}, sortField={{ .Get "sortField" }}, scaleChart={{ .Get "scaleChart" }})
+});
+</script>
+</div>
+</section>

layouts/shortcodes/dropdown_filter.html

@@ -0,0 +1,29 @@
+{{ $id := .Get "id" }}
+{{ $default_selection := .Get "default_selection" }}
+{{ $options := .Get "options" }}
+
+{{ $options_split := split $options "," }}
+
+<div class="dropdown-filter-container">
+  <select class="filter dropdown-filter" id="{{ $id }}" idFilter='{{ .Get "id_filter" }}' onchange="dispatchDropdownEvent(this)">
+  {{ range $options_split }}
+    {{ $parts := split . ":" }}
+    {{ $key := index $parts 0 }}
+    {{ $value := index $parts 1 }}
+    <option value="{{ $value }}" {{ if eq $key $default_selection }}selected{{ end }}>{{ $key }}</option>
+  {{ end }}
+</select>
+<script>
+  function dispatchDropdownEvent(selectElement) {
+    const event = new CustomEvent('filterChange', {
+      detail: {
+        filterId: '{{ .Get "id_filter" }}',
+        filterValue: selectElement.value,
+        filterActions: ["refresh"],
+        filterTargets: '{{ .Get "targets" }}'.split(" ")
+      }
+    });
+    document.dispatchEvent(event);
+  }
+</script>
+</div>

layouts/shortcodes/table.html

@@ -1,67 +1,8 @@
+{{ partial "table.html" }}
+<div id = '{{ .Get "id" }}--container'>
 <script>
-  async function fetchDataForTable() {
-    try {
-      const apiEndpoint = `${apiURL}/{{ .Get "endpoint" }}`;
-      const response = await fetch(apiEndpoint);
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
-      const fetchedData = await response.json();
-      data = fetchedData;
-      console.log(data);
-      createTable(data);
-    } catch (error) {
-      console.error("Fetching data failed:", error);
-    }
-  }
-
-  function createTable(data) {
-    const jsonTableContainer = document.getElementById("jsonTableContainer");
-    jsonTableContainer.innerHTML = "";
-    jsonTableContainer.style.maxHeight = "{{ .Get `maxHeight` }}"
-
-    tableHeaderNames = Object.values({{ .Get `headers` | safeJS }});
-    tableHeaderKeys = Object.keys({{ .Get `headers` | safeJS }});
-
-    const table = document.createElement("table");
-    table.id = "{{ .Get `id` }}"
-    const thead = document.createElement("thead");
-    const tbody = document.createElement("tbody");
-    const headerRow = document.createElement("tr");
-    tableHeaderNames.forEach((header) => {
-      const th = document.createElement("th");
-      th.textContent = header;
-      headerRow.appendChild(th);
-    });
-    thead.appendChild(headerRow);
-    table.appendChild(thead);
-
-    for (const key in data) {
-      const row = document.createElement('tr');
-      tableHeaderKeys.forEach((columnName) => {
-        const td = document.createElement("td");
-        const div = document.createElement("div");
-        div.id = "scrollable";
-        if (columnName == "{{ .Get `tableKey` }}") {
-          div.textContent = key;
-        } else {
-          div.textContent = data[key][columnName];
-        };
-        td.appendChild(div);
-        row.appendChild(td);
-        tbody.appendChild(row);
-      });
-    }
-    table.appendChild(thead);
-    table.appendChild(tbody);
-    jsonTableContainer.appendChild(table)
-    {{ if eq (.Get "sortable") "true" }}
-      table.className = "sortable"
-      sorttable.makeSortable(document.getElementById("{{ .Get `id` }}"));
-    {{ end }}
-
-    table.className = ""
-  }
+document.addEventListener("DOMContentLoaded", function () {
+  createTable({{ .Get "endpoint" }}, {{ .Get "id" }}, {{ .Get "headers" | safeJS }}, {{ .Get "maxHeight" }}, {{ .Get "sortable" }}, {{ .Get "valueId" }}, {{ .Get "selectableRows" }}, '{{ .Get "targets" }}'.split(" "), {{ .Get "defaultFirstSelected" | safeJS }})
+});
 </script>
-<div id="jsonTableContainer"></div>
-<script src="/js/lib/sorttable.js"></script>
+</div>

pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "baseddata-io"
+version = "0.1.0"
+description = ""
+authors = ["Sam <samual.shop@proton.me>"]
+readme = "README.md"
+packages = [{include = "baseddata"}]
+package-mode = false
+
+[virtualenvs]
+in-project = true
+
+[tool.poetry.dependencies]
+python = "^3.11"
+fastapi = "^0.115.4"
+uvicorn = "^0.32.0"
+psycopg2 = "^2.9.10"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

shell.nix

@@ -1,20 +1,19 @@
-{ pkgs ? import <nixpkgs> { } }:
-
+{pkgs ? import <nixpkgs> {}}:
 pkgs.mkShell
 {
   nativeBuildInputs = with pkgs; [
-    python312Packages.flask
-    python312Packages.flask-cors
-    python312Packages.requests
-    python312Packages.pandas
-    python312Packages.orjson
     hugo
-    tmux
+    python311
+    poetry
   ];
 
-  shellHook = '' 
+  NIX_LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib;
+  NIX_LD=/run/current-system/sw/share/nix-ld/lib/ld.so;
+  shellHook = ''
     ${pkgs.cowsay}/bin/cowsay "Welcome to the baseddata.io development environment!" | ${pkgs.lolcat}/bin/lolcat
-
+    export LD_LIBRARY_PATH=$NIX_LD_LIBRARY_PATH
+    source .env
+    source .venv/bin/activate
     get_session=$(tmux list-session | grep "baseddata")
 
     if [ -z "$get_session" ];
@@ -22,9 +21,8 @@ pkgs.mkShell
        tmux new-session -d -s baseddata
        tmux split-window -h
        tmux send-keys -t 0 "hugo server" C-m
-       tmux send-keys -t 1 "cd backend && python app.py" C-m
+       tmux send-keys -t 1 "cd backend && uvicorn main:app --reload" C-m
        echo "Baseddata running in dev tmux shell"
     fi
-    '';
+  '';
 }
-

static/css/charts.css

@@ -1,24 +1,12 @@
 /* Charts */
-.chart-flex-container {
-  display: flex;
-}
-
-.chart-flex-container article {
-  flex: 1;
-}
-
-#chart {
-  width: 100%;
-  aspect-ratio: 16 / 9;
-}
-
 .chart-container {
-  margin-top: 20px;
   display: flex;
-  justify-content: center;
-  flex-direction: column;
+  /* height: 600px; */
+  aspect-ratio: 1 / 1;
 }
 
-#chart-modifiers {
+.chart {
   display: flex;
+  height: 100%;
+  width: 100%;
 }

static/css/tables.css

@@ -1,9 +1,13 @@
 /* Tables */
-#jsonTableContainer {
+.jsonTableContainer {
   display: flex;
   overflow-y: auto;
 }
 
+table.sortable th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after { 
+    content: " \25B4\25BE" 
+}
+
 table {
   width: 100%;
   border-collapse: collapse;
@@ -49,3 +53,10 @@ tr:nth-child(odd) {
   background-color: var(--table-odd-row-bg-color);
   font-size: var(--table-row-font-size);
 }
+tr:hover {
+  background-color: #f5f5f5;
+}
+
+tr.selected {
+  background-color: #d1ecf1;
+}

static/js/lib/helper-functions.js

@@ -0,0 +1,16 @@
+function queryConstructor(customFilters = {}) {
+  let filters = document.querySelectorAll(".filter");
+  let queryObject = {};
+
+  Object.assign(queryObject, customFilters);
+
+  filters.forEach((filter) => {
+    const filterId = filter.getAttribute("idFilter");
+    const filterValue = filter.value;
+    queryObject[filterId] = filterValue;
+  });
+
+  let queryString = `query=${JSON.stringify(queryObject)}`;
+
+  return queryString;
+}

static/js/mangrove-map.js

@@ -15,6 +15,3 @@ map.on("load", () => {
   });
 });
 
-document.addEventListener("DOMContentLoaded", function () {
-  fetchDataForTable();
-});