From ad530b133f3cc819ef8d2024926cf193c64e792d Mon Sep 17 00:00:00 2001 From: Oleh Omelchenko Date: Thu, 13 Mar 2025 12:11:55 +0200 Subject: [PATCH] init --- .DS_Store | Bin 0 -> 8196 bytes .gitignore | 5 ++ .vscode/extensions.json | 7 ++ README.md | 113 +++++++++++++++++++++++++ analyses/.gitkeep | 0 dbt_project.yml | 36 ++++++++ macros/.gitkeep | 0 models/example/my_first_dbt_model.sql | 27 ++++++ models/example/my_second_dbt_model.sql | 6 ++ models/example/schema.yml | 21 +++++ profiles.yml | 22 +++++ requirements.txt | 3 + seeds/.gitkeep | 0 snapshots/.gitkeep | 0 tests/.gitkeep | 0 15 files changed, 240 insertions(+) create mode 100644 .DS_Store create mode 100644 .gitignore create mode 100644 .vscode/extensions.json create mode 100644 README.md create mode 100644 analyses/.gitkeep create mode 100644 dbt_project.yml create mode 100644 macros/.gitkeep create mode 100644 models/example/my_first_dbt_model.sql create mode 100644 models/example/my_second_dbt_model.sql create mode 100644 models/example/schema.yml create mode 100644 profiles.yml create mode 100644 requirements.txt create mode 100644 seeds/.gitkeep create mode 100644 snapshots/.gitkeep create mode 100644 tests/.gitkeep diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..82523ad422eb359131cb192ea4d29047de76407b GIT binary patch literal 8196 zcmeHM-AcnS6h6^TDjgN!R`cM?uSj->~K>MM0({jc_n(GFXK!2og~bp zO?D&5z!ONmlJh0!+t<)?IwDftyHSVen21tz#`YnGK;t;~M%%G78Bk$8%_g1Ru$(U% zOqmb|!~tqeMDgJwA?TYM;+S=kCj>Fn4)({xg~K^esX zaiHlyhN*;3XhPSt^6U4fS2s256X9XY>G7V_Mb}vE~LRqA9=& zs%r3jU7V8kXbx3Fx`j_6O2nvUei>RiV+=x{hu9czxSAN$=nW?GZpc{>7aMnl8io@{p>kJ3T&*x+ToZ_CWf;s}Mh@K!b=uCha(AR<@oZg5o zCmPY?F@icAMl-*>|Me-6K^)j_2X>9TW3K;g%dPNso*xI_W v8Q0nwdLNw|^D2XS2?nO{ye`Lq>pu)}o!MA!u`-AVntTY*Hpn0j{HX)q8+Hh2 literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce1e6a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ + +target/ +dbt_packages/ +logs/ +env/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..53712e5 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "innoverio.vscode-dbt-power-user", + "mhutchie.git-graph", + "ms-python.python" + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..1e0aa6f --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +Welcome to your new dbt project! + +# Installation + +The following tutorial assumes you're already familiar with git and command line usage. + +## Getting the code to your local machine +1. Fork this github repository into your local account + +2. Copy it to your local machine: `git clone https://github.com/your_account_name/econ250_2025.git` + + + +## gcloud authentication + +To run queries from your command line, you'll first need to install `gcloud` utility. + +Follow the instructions here: https://cloud.google.com/sdk/docs/install. After installation you should have `gcloud` command available for running in the terminal. + +Now, try to authenticate with your **kse email** using the following command: + +```bash +gcloud auth application-default login \ + --scopes=https://www.googleapis.com/auth/bigquery,\ +https://www.googleapis.com/auth/drive.readonly,\ +https://www.googleapis.com/auth/iam.test +``` + +Now, when you run the following commands something similar should be response: + +```bash +$ gcloud auth list + + Credentialed Accounts +ACTIVE ACCOUNT +* o_omelchenko@kse.org.ua + +``` +To set the active project, run the following: + +```bash +gcloud config set project econ250-2025 +``` + + +## venv and libraries +Prerequisites: having Python installed on your machine. +Following instructions are for Linux or WSL; if you'd like to run Windows - please refer to the documentation below. + +```bash + +# change directory to the one you just copied from github +cd econ250_2025 + +# create and activate venv +python3 -m venv env +source env/bin/activate + +pip install -r requirements.txt + +``` + +If everything is installed correctly, you should run the following commands successfully: + + +``` +$ dbt --version + +Core: + - installed: 1.9.3 + - latest: 1.9.3 - Up to date! + +Plugins: + - bigquery: 1.9.1 - Up to date! +``` + + +For more detailed reference, refer to the official documentation here: +- https://docs.getdbt.com/docs/core/pip-install +- https://docs.getdbt.com/docs/core/connect-data-platform/bigquery-setup#local-oauth-gcloud-setup + +## Adjusting the configuration + +You'll need to specify your own dataset to save your models to. To do so, navigate to the `profiles.yml` in the root directory of the project, and replace `o_omelchenko` with your bigquery dataset name with which you have been working previously. + + + + +## Final check + +Try running the following command: +- dbt run + +If everything is set up well, you will see similar output: + +```log +❯ dbt run +01:18:56 Running with dbt=1.9.3 +01:18:57 Registered adapter: bigquery=1.9.1 +01:18:57 Found 2 models, 4 data tests, 491 macros +01:18:57 +01:18:57 Concurrency: 2 threads (target='dev') +01:18:57 +01:19:00 1 of 2 START sql table model o_omelchenko.my_first_dbt_model ................... [RUN] +01:19:04 1 of 2 OK created sql table model o_omelchenko.my_first_dbt_model .............. [CREATE TABLE (2.0 rows, 0 processed) in 4.44s] +01:19:04 2 of 2 START sql view model o_omelchenko.my_second_dbt_model ................... [RUN] +01:19:06 2 of 2 OK created sql view model o_omelchenko.my_second_dbt_model .............. [CREATE VIEW (0 processed) in 2.13s] +01:19:06 +01:19:06 Finished running 1 table model, 1 view model in 0 hours 0 minutes and 9.64 seconds (9.64s). +``` + +If you have any troubles with installation, please contact the course instructor (Oleh Omelchenko) in slack for assist. + diff --git a/analyses/.gitkeep b/analyses/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..43ac96b --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,36 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'econ250_2025' +version: '1.0.0' + +# This setting configures which "profile" dbt uses for this project. +profile: 'econ250_2025' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + econ250_2025: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: view diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/models/example/my_first_dbt_model.sql b/models/example/my_first_dbt_model.sql new file mode 100644 index 0000000..f31a12d --- /dev/null +++ b/models/example/my_first_dbt_model.sql @@ -0,0 +1,27 @@ + +/* + Welcome to your first dbt model! + Did you know that you can also configure models directly within SQL files? + This will override configurations stated in dbt_project.yml + + Try changing "table" to "view" below +*/ + +{{ config(materialized='table') }} + +with source_data as ( + + select 1 as id + union all + select null as id + +) + +select * +from source_data + +/* + Uncomment the line below to remove records with null `id` values +*/ + +-- where id is not null diff --git a/models/example/my_second_dbt_model.sql b/models/example/my_second_dbt_model.sql new file mode 100644 index 0000000..c91f879 --- /dev/null +++ b/models/example/my_second_dbt_model.sql @@ -0,0 +1,6 @@ + +-- Use the `ref` function to select from other models + +select * +from {{ ref('my_first_dbt_model') }} +where id = 1 diff --git a/models/example/schema.yml b/models/example/schema.yml new file mode 100644 index 0000000..9730b70 --- /dev/null +++ b/models/example/schema.yml @@ -0,0 +1,21 @@ + +version: 2 + +models: + - name: my_first_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + data_tests: + - unique + - not_null + + - name: my_second_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + data_tests: + - unique + - not_null diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..60e8562 --- /dev/null +++ b/profiles.yml @@ -0,0 +1,22 @@ +econ250_2025: + outputs: + dev: + dataset: o_omelchenko # TODO: Change this to your personal dataset + job_execution_timeout_seconds: 300 + job_retries: 1 + location: US + method: oauth + priority: interactive + project: econ250-2025 + threads: 2 + type: bigquery + target: dev +my-bigquery-db: + outputs: + dev: + dataset: o_omelchenko # TODO: Change this to your personal dataset + method: oauth + project: econ250-2025 + threads: 4 + type: bigquery + target: dev diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2231627 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +dbt-bigquery==1.9.1 +dbt-core==1.9.3 +google-cloud-core==2.4.3 diff --git a/seeds/.gitkeep b/seeds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29