From c64e22f007f92935730375cb1019ec1d3137b01f Mon Sep 17 00:00:00 2001 From: Sam Date: Sat, 11 Jan 2025 15:10:53 +0000 Subject: [PATCH] major refactor --- backend/api/pipelines.py | 39 +++ backend/api/route.py | 42 +++ backend/api/schemas.py | 26 ++ backend/baseddata.service | 2 +- backend/requirements.txt | 5 - .../data-lab/{price.md => bitcoin-price.md} | 6 +- ...-distribution.md => feerate-percentile.md} | 6 +- .../global-bitcoin-business-growth.md | 23 ++ content/data-lab/global-business-growth.md | 30 -- content/data-lab/hashrate.md | 4 +- content/data-lab/miner-rewards.md | 5 +- content/data-lab/protected-mangroves.md | 1 + layouts/data-lab/baseof.html | 7 +- layouts/partials/chart.html | 162 ---------- layouts/partials/head.html | 5 - layouts/partials/table.html | 152 ---------- layouts/shortcodes/bitcoin-price.html | 19 -- layouts/shortcodes/chart.html | 8 +- layouts/shortcodes/dropdown_filter.html | 31 +- layouts/shortcodes/js.html | 9 + layouts/shortcodes/table.html | 11 +- static/js/bitcoin-business-growth-chart.js | 102 ------- static/js/bitcoin-business-growth-table.js | 95 ------ static/js/bitcoin-price.js | 84 ------ static/js/chart-params.js | 139 --------- static/js/content/bitcoin-price.js | 93 ++++++ static/js/content/feerate-percentile.js | 106 +++++++ .../content/global-bitcoin-business-growth.js | 279 ++++++++++++++++++ .../global-mangroves-protected-areas.md} | 0 static/js/content/hashrate.js | 261 ++++++++++++++++ static/js/{ => content}/miner-rewards.js | 61 ++-- static/js/feerate-percentile.js | 127 -------- static/js/hashrate.js | 127 -------- static/js/lib/chart-params.js | 90 ++++++ static/js/lib/echarts.min.js | 45 +++ static/js/lib/helper-functions.js | 50 +++- 36 files changed, 1129 insertions(+), 1123 deletions(-) delete mode 100644 backend/requirements.txt rename content/data-lab/{price.md => bitcoin-price.md} (59%) rename content/data-lab/{feerate-distribution.md => feerate-percentile.md} (58%) create mode 100644 content/data-lab/global-bitcoin-business-growth.md delete mode 100644 content/data-lab/global-business-growth.md delete mode 100644 layouts/partials/chart.html delete mode 100644 layouts/partials/table.html delete mode 100644 layouts/shortcodes/bitcoin-price.html create mode 100644 layouts/shortcodes/js.html delete mode 100644 static/js/bitcoin-business-growth-chart.js delete mode 100644 static/js/bitcoin-business-growth-table.js delete mode 100644 static/js/bitcoin-price.js delete mode 100644 static/js/chart-params.js create mode 100644 static/js/content/bitcoin-price.js create mode 100644 static/js/content/feerate-percentile.js create mode 100644 static/js/content/global-bitcoin-business-growth.js rename static/js/{mangrove-map.js => content/global-mangroves-protected-areas.md} (100%) create mode 100644 static/js/content/hashrate.js rename static/js/{ => content}/miner-rewards.js (69%) delete mode 100644 static/js/feerate-percentile.js delete mode 100644 static/js/hashrate.js create mode 100644 static/js/lib/chart-params.js create mode 100644 static/js/lib/echarts.min.js diff --git a/backend/api/pipelines.py b/backend/api/pipelines.py index 8975ed8..fc7a4ca 100644 --- a/backend/api/pipelines.py +++ b/backend/api/pipelines.py @@ -93,3 +93,42 @@ def miner_rewards(args): order by date asc """ + +def feerate_percentiles(args): + days_ago = parse_int(args["days_ago"]) + return f""" + with + filtered_data as ( + select * from models_final.final__feerate_percentiles order by date desc limit {days_ago} + ) + select * + from filtered_data + order by date asc + +""" + +def bitcoin_price_timeseries(args): + days_ago = parse_int(args["days_ago"]) + return f""" + with + filtered_data as ( + select * from models_final.final__bitcoin_price order by date desc limit {days_ago} + ) + select * + from filtered_data + order by date asc + +""" + +def bitcoin_hashrate(args): + days_ago = parse_int(args["days_ago"]) + return f""" + with + filtered_data as ( + select * from models_final.final__hashrate order by date desc limit {days_ago} + ) + select * + from filtered_data + order by date asc + +""" diff --git a/backend/api/route.py b/backend/api/route.py index dac656b..7a12f2b 100644 --- a/backend/api/route.py +++ b/backend/api/route.py @@ -79,3 +79,45 @@ async def miner_rewards(query: str): rawData = handler.execute_query(pipeline) serializedData = serializer.serialize_many(rawData) return serializedData + +@router.get("/feerate_percentiles") +async def feerate_percentiles(query: str): + args = parse_args_to_dict(query) + + pipeline = pipelines.feerate_percentiles(args) + handler = PostgresHandler() + + schema = schemas.feerate_percentiles_schema + serializer = DataSerializer(schema) + + rawData = handler.execute_query(pipeline) + serializedData = serializer.serialize_many(rawData) + return serializedData + +@router.get("/bitcoin_price_timeseries") +async def bitcoin_price_timeseries(query: str): + args = parse_args_to_dict(query) + + pipeline = pipelines.bitcoin_price_timeseries(args) + handler = PostgresHandler() + + schema = schemas.bitcoin_price_timeseries_schema + serializer = DataSerializer(schema) + + rawData = handler.execute_query(pipeline) + serializedData = serializer.serialize_many(rawData) + return serializedData + +@router.get("/bitcoin_hashrate") +async def bitcoin_hashrate(query: str): + args = parse_args_to_dict(query) + + pipeline = pipelines.bitcoin_hashrate(args) + handler = PostgresHandler() + + schema = schemas.bitcoin_hashrate_schema + serializer = DataSerializer(schema) + + rawData = handler.execute_query(pipeline) + serializedData = serializer.serialize_many(rawData) + return serializedData diff --git a/backend/api/schemas.py b/backend/api/schemas.py index 698bcf6..110020a 100644 --- a/backend/api/schemas.py +++ b/backend/api/schemas.py @@ -51,6 +51,32 @@ def miner_rewards_schema(data): "subsidy_usd": data["subsidy_usd"], } +def feerate_percentiles_schema(data): + return { + "date": data["date"], + "feerate_10th": data["feerate_10th"], + "feerate_25th": data["feerate_25th"], + "feerate_50th": data["feerate_50th"], + "feerate_75th": data["feerate_75th"], + "feerate_90th": data["feerate_90th"], + "maxfeerate": data["maxfeerate"], + "minfeerate": data["minfeerate"], + } + +def bitcoin_price_timeseries_schema(data): + return { + "date": data["date"], + "price": data["price"], + } + +def bitcoin_hashrate_schema(data): + return { + "date": data["date"], + "hashrate": data["hashrate"], + "difficulty": data["difficulty"], + "hashrate28": data["hashrate28"], + "difficulty28": data["difficulty28"], + } class DataSerializer: def __init__(self, schema_func): diff --git a/backend/baseddata.service b/backend/baseddata.service index b0ff100..ebc33eb 100644 --- a/backend/baseddata.service +++ b/backend/baseddata.service @@ -7,7 +7,7 @@ User=admin Group=www-data WorkingDirectory=/var/www/baseddata.io/backend Environment="PATH=/var/www/baseddata.io/.venv/bin" -ExecStart=/var/www/baseddata.io/.venv/bin/gunicorn --workers 4 --bind unix:baseddata.sock -m 007 app:app +ExecStart=/var/www/baseddata.io/.venv/bin/gunicorn --workers 4 --bind unix:baseddata.sock -m 007 main:app [Install] WantedBy=multi-user.target diff --git a/backend/requirements.txt b/backend/requirements.txt deleted file mode 100644 index 88139ee..0000000 --- a/backend/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -flask -flask_cors -orjson -gunicorn - diff --git a/content/data-lab/price.md b/content/data-lab/bitcoin-price.md similarity index 59% rename from content/data-lab/price.md rename to content/data-lab/bitcoin-price.md index 502661f..90401ad 100644 --- a/content/data-lab/price.md +++ b/content/data-lab/bitcoin-price.md @@ -6,11 +6,11 @@ author: header_image: "/pics/charts/price.webp" summary: "Daily bitcoin price. Data is obtained from CoinGecko using their public API." tags: ["Bitcoin", "Stats"] +jsScripts: [ "/js/lib/chart-params.js", "/js/lib/helper-functions.js", "/js/lib/echarts.min.js", "/js/lib/sorttable.js", "/js/content/bitcoin-price.js"] --- Daily bitcoin price. Data is obtained from [CoinGecko](https://www.coingecko.com/) using their public API. -{{< bitcoin-price >}} - -{{< chart src="/js/bitcoin-price.js" >}} +{{< dropdown_filter id="days_ago_dropdown_filter" >}} +{{< chart id = "bitcoin-price-chart" >}} diff --git a/content/data-lab/feerate-distribution.md b/content/data-lab/feerate-percentile.md similarity index 58% rename from content/data-lab/feerate-distribution.md rename to content/data-lab/feerate-percentile.md index f215fee..57f7fd9 100644 --- a/content/data-lab/feerate-distribution.md +++ b/content/data-lab/feerate-percentile.md @@ -5,10 +5,12 @@ author: name: "Sam Chance" header_image: "/pics/charts/feerate-percentile.webp" summary: "Bar chart showing historical median daily feerate percentiles for the Bitcoin protocol." -chart: "/js/feerate_percentile.js" tags: ["Bitcoin", "Stats"] +jsScripts: [ "/js/lib/chart-params.js", "/js/lib/helper-functions.js", "/js/lib/echarts.min.js", "/js/lib/sorttable.js", "/js/content/feerate-percentile.js"] --- This chart shows historical median daily feerate percentiles for the Bitcoin protocol. -{{< chart src="/js/feerate-percentile.js" >}} +{{< dropdown_filter id="days_ago_dropdown_filter" >}} + +{{< chart id="feerate-percentiles-chart" >}} diff --git a/content/data-lab/global-bitcoin-business-growth.md b/content/data-lab/global-bitcoin-business-growth.md new file mode 100644 index 0000000..8b1a096 --- /dev/null +++ b/content/data-lab/global-bitcoin-business-growth.md @@ -0,0 +1,23 @@ +--- +title: "Global Bitcoin Business Growth" +date: 2024-04-04T09:16:33+01:00 +author: + name: "Sam Chance" +header_image: "/pics/charts/growth.webp" +summary: "This analysis uses OpenStreetMaps data to chart the yearly growth of bitcoin-accepting businesses worldwide." +tags: ["Bitcoin", "Stats", "OpenStreetMaps"] +jsScripts: [ "/js/lib/chart-params.js", "/js/lib/helper-functions.js", "/js/lib/echarts.min.js", "/js/lib/sorttable.js", "/js/content/global-bitcoin-business-growth.js"] +--- + +The table below illustrates growth of businesses worldwide that accept bitcoin as payment for products or services. The accompanying chart displays the cumulative number of bitcoin-accepting businesses for the countries selected in the table. The data is sourced from OpenStreetMaps. + +You can select the period of interest from the drop-down, which updates the values in the table. The table shows the ***Previous*** value, which was the number of businesses `n days ago` specified in the drop-down. The ***Current*** value is the number of businesses as of the latest update. The table also shows the ***Absolute Difference***, and the ***Percent Difference*** between these periods. + +
+{{< dropdown_filter id="days_ago_dropdown_filter" >}} + +{{< table id="bitcoin-business-growth-table" >}} +{{< chart id="bitcoin-business-growth-chart" >}} + +#### Attribution +Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright) diff --git a/content/data-lab/global-business-growth.md b/content/data-lab/global-business-growth.md deleted file mode 100644 index 954aab9..0000000 --- a/content/data-lab/global-business-growth.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: "Global Bitcoin Business Growth" -date: 2024-04-04T09:16:33+01:00 -author: - name: "Sam Chance" -header_image: "/pics/charts/growth.webp" -summary: "This analysis uses OpenStreetMaps data to chart the yearly growth of bitcoin-accepting businesses worldwide." -tags: ["Bitcoin", "Stats", "OpenStreetMaps"] ---- - -The table below illustrates growth of businesses worldwide that accept bitcoin as payment for products or services. The accompanying chart displays the cumulative number of bitcoin-accepting businesses for the countries selected in the table. The data is sourced from OpenStreetMaps and is updated approximately every 2 hours. - -You can select the growth period of interest from the drop-down, which updates the values in the table. The table shows the ***Previous*** value, which was the number of businesses *n* days ago specified in the drop-down. The ***Current*** value is the number of businesses as of the latest update. The table also shows the ***Absolute Difference***, and the ***Percent Difference*** between this period. - -The chart always reflects the countries selected in the table. - -
-{{< dropdown_filter id="days_ago_dropdown_filter" id_filter="days_ago" options="1 day:1,7 day:7,28 day:28,1 year:365,5 year:1826,10 year:3652,all time:10000" default_selection="7 day" targets="bitcoin-business-growth-chart bitcoin-business-growth-table" >}} -{{< table id="bitcoin-business-growth-table" endpoint="bitcoin_business_growth_percent_diff" headers="{'country_name': 'Country', 'date_range': 'Date Range', 'first_value': 'Previous #', 'last_value': 'Current #', 'difference': 'Diff', 'percent_difference': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_name" selectableRows="multi" targets="bitcoin-business-growth-chart" defaultFirstSelected="true" >}} - -{{< chart id="bitcoin-business-growth-table" src="/js/bitcoin-business-growth-chart.js" >}} - -{{< chart chartMethod="tableRowSelectChart" id="bitcoin-business-growth-chart" endpoint="bitcoin_business_growth_timeseries" chartType="line" xAxisField="date" yAxisField="cumulative_value" scaleChart=true xAxisType="category" >}} - -#### Attribution and License -Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright) - -Licensed under the [ODbl](https://opendatacommons.org/licenses/odbl/) - - diff --git a/content/data-lab/hashrate.md b/content/data-lab/hashrate.md index 872b9d5..c059454 100644 --- a/content/data-lab/hashrate.md +++ b/content/data-lab/hashrate.md @@ -6,8 +6,10 @@ author: header_image: "/pics/charts/hashrate.webp" summary: "Timeseries chart showing the Bitcoin network hashrate and difficulty." tags: ["Bitcoin", "Stats", "Hashrate"] +jsScripts: [ "/js/lib/chart-params.js", "/js/lib/helper-functions.js", "/js/lib/echarts.min.js", "/js/lib/sorttable.js", "/js/content/hashrate.js"] --- This chart shows the estimated hashrate and difficulty of the Bitcoin network, accompanied by the 28-day moving average. This information is extracted from a bitcoin node using the `bitcoin-cli getnetworkhashps` command. -{{< chart src="/js/hashrate.js" >}} +{{< dropdown_filter id="days_ago_dropdown_filter" >}} +{{< chart id = "hashrate-chart" >}} diff --git a/content/data-lab/miner-rewards.md b/content/data-lab/miner-rewards.md index e0e522e..38f9ba8 100644 --- a/content/data-lab/miner-rewards.md +++ b/content/data-lab/miner-rewards.md @@ -7,10 +7,11 @@ summary: "Miner rewards" header_image: "/pics/charts/rewards.webp" draft: false tags: ["Bitcoin", "Stats"] +jsScripts: [ "/js/lib/chart-params.js", "/js/lib/helper-functions.js", "/js/lib/echarts.min.js", "/js/lib/sorttable.js", "/js/content/miner-rewards.js"] --- The following chart shows daily miner revenue in USD for the period selected in the dropdown. This information is based on the sum of bitcoin mined each day (i.e. the block-subsidy) plus the transaction fees. Price data is obtained from [CoinGecko](https://www.coingecko.com/). -{{< dropdown_filter id="days_ago_dropdown_filter" id_filter="days_ago" options="1 day:1,7 day:7,28 day:28,1 year:365,5 year:1826,10 year:3652,all time:10000" default_selection="7 day" targets="miner-rewards-chart" >}} +{{< dropdown_filter id="days_ago_dropdown_filter" >}} -{{< chart id="miner-rewards" src="/js/miner-rewards.js" >}} +{{< chart id="miner-rewards-chart" >}} diff --git a/content/data-lab/protected-mangroves.md b/content/data-lab/protected-mangroves.md index 47d935d..93ae275 100644 --- a/content/data-lab/protected-mangroves.md +++ b/content/data-lab/protected-mangroves.md @@ -7,6 +7,7 @@ header_image: "/pics/charts/price.webp" summary: "Daily bitcoin price. Data is obtained from CoinGecko using their public API." tags: ["Bitcoin", "Stats"] script: "/js/mangrove-map.js" +draft: true --- {{< table id="mangrove_countries" endpoint="mangrove_by_country_latest" headers="{'country_with_parent': 'Country', 'original_pixels': '1996 Cover', 'total_n_pixels': '2020 Cover', 'cumulative_pixels_diff': 'Diff', 'cumulative_pct_diff': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_with_parent" selectableRows="single" defaultFirstSelected="true" targets="mangrove-country-timeseries-chart" >}} diff --git a/layouts/data-lab/baseof.html b/layouts/data-lab/baseof.html index ce96df4..cfe7db8 100644 --- a/layouts/data-lab/baseof.html +++ b/layouts/data-lab/baseof.html @@ -1,6 +1,11 @@ - + {{ partial "head.html" . }} {{ template "partials/body.html" . }} +{{ range .Params.jsScripts }} + +{{ end }} diff --git a/layouts/partials/chart.html b/layouts/partials/chart.html deleted file mode 100644 index 1e94b9e..0000000 --- a/layouts/partials/chart.html +++ /dev/null @@ -1,162 +0,0 @@ - - diff --git a/layouts/partials/head.html b/layouts/partials/head.html index 5624d97..1e271c0 100644 --- a/layouts/partials/head.html +++ b/layouts/partials/head.html @@ -8,8 +8,6 @@ - - - diff --git a/layouts/partials/table.html b/layouts/partials/table.html deleted file mode 100644 index 2a21d0a..0000000 --- a/layouts/partials/table.html +++ /dev/null @@ -1,152 +0,0 @@ - diff --git a/layouts/shortcodes/bitcoin-price.html b/layouts/shortcodes/bitcoin-price.html deleted file mode 100644 index 9e8a52e..0000000 --- a/layouts/shortcodes/bitcoin-price.html +++ /dev/null @@ -1,19 +0,0 @@ - - - -

- - - - diff --git a/layouts/shortcodes/chart.html b/layouts/shortcodes/chart.html index 4087315..16c5a28 100644 --- a/layouts/shortcodes/chart.html +++ b/layouts/shortcodes/chart.html @@ -1,7 +1,3 @@ - -
- -
- -
+
+
diff --git a/layouts/shortcodes/dropdown_filter.html b/layouts/shortcodes/dropdown_filter.html index d70c1b6..7b6da0a 100644 --- a/layouts/shortcodes/dropdown_filter.html +++ b/layouts/shortcodes/dropdown_filter.html @@ -1,29 +1,4 @@ -{{ $id := .Get "id" }} -{{ $default_selection := .Get "default_selection" }} -{{ $options := .Get "options" }} - -{{ $options_split := split $options "," }} - -