feat(); SPL Tokens; new substream

Cargo.toml

@@ -12,13 +12,14 @@ members = [
     "uniswap-v2",
     "solana-sample",
     "ens-names",
+    "spl-balance",
 ]
 exclude = ["messari-cli"]
 
 [workspace.dependencies]
 substreams = "0.3.2"
 substreams-ethereum = "0.6.2"
-substreams-solana = "0.1.0"
+substreams-solana = "0.2.0"
 
 [build]
 target = "wasm32-unknown-unknown"

Makefile

@@ -11,6 +11,7 @@ build-all:
 	$(MAKE) -C solana-sample build
 	$(MAKE) -C eth-balance build
 	$(MAKE) -C ens-names build
+	$(MAKE) -C spl-balance build
 
 .PHONY: run-all
 run-all:
@@ -20,6 +21,7 @@ run-all:
 	$(MAKE) -C erc721 run
 	$(MAKE) -C network run
 	$(MAKE) -C ens-names run
+	$(MAKE) -C spl-balance run
 
 .PHONY: test
 test:

README.md

@@ -33,16 +33,16 @@
 🛠 = Feature complete. Additional testing required.  
 ✅ = Production-ready.
 
-| Substream        |  Status | Description                                |
-|------------------|  :------: |--------------------------------------------|
-| Ethereum Network | 🔨 | Network-level metrics and statistics       |
-| ETH Balance      | 🛠 | ETH balance for every Ethereum address     |
-| ERC20 Holdings   | 🛠 | ERC20 balance for every Ethereum address   |
-| ERC20 Price      | 🛠 | Pricing module for ERC20 tokens            |
-| SPL Holdings     |  | SPL token balance for every Solana address |
-| ENS Look Up      | 🔨 | ENS records for lookup and reverse lookup  |
-| Uniswap v2       | 🔨 | Substreams for Uniswap v2                  |
-| Compound v2      | 🔨 | Substreams for Compound v2                 |
+| Substream        | Status | Description                                |
+|------------------|:------:|--------------------------------------------|
+| Ethereum Network |   🔨   | Network-level metrics and statistics       |
+| ETH Balance      |   🛠   | ETH balance for every Ethereum address     |
+| ERC20 Holdings   |   🛠   | ERC20 balance for every Ethereum address   |
+| ERC20 Price      |   🛠   | Pricing module for ERC20 tokens            |
+| SPL Holdings     |   🛠   | SPL token balance for every Solana address |
+| ENS Look Up      |   🔨   | ENS records for lookup and reverse lookup  |
+| Uniswap v2       |   🔨   | Substreams for Uniswap v2                  |
+| Compound v2      |   🔨   | Substreams for Compound v2                 |
 
 ## Workflow
 

common/proto/sol_token.proto

@@ -0,0 +1,47 @@
+syntax = "proto3";
+
+package messari.sol_token.v1;
+
+message Transfers {
+  repeated Transfer transfers = 1;
+}
+
+message Transfer {
+  string signature = 1;
+  string from = 2;
+  string to = 3;
+  TokenAccount token = 4;
+  string mint = 5;
+  uint64 native_amount = 6;
+  uint64 amount = 7;
+  BalanceChanges balance_changes = 8;
+}
+
+message BalanceChanges {
+  repeated TokenBalance items = 1;
+}
+
+// balance change
+message TokenBalance {
+  TokenAccount token = 1;
+  string transaction_id = 2;
+  uint64 block_height = 3;
+  string address = 4; // account address of the balance change
+  string pre_balance = 5; // BigInt, in token's native amount
+  string post_balance = 6; // BigInt, in token's native amount
+}
+
+message Tokens {
+  repeated TokenAccount tokens = 1;
+}
+
+message TokenAccount {
+  string address = 1;
+  string name = 2;
+  string symbol = 3;
+  uint64 decimals = 4;
+  string freeze_authority = 5;
+  string mint_authority = 6;
+  string tx_created = 7;
+  uint64 block_created = 8;
+}

solana-sample/Cargo.toml

@@ -5,17 +5,17 @@ description = "Sample substreams for Solana"
 edition = "2021"
 repository = "https://github.com/messari/substreams/solana-sample"
 
+[lib]
+crate-type = ["cdylib"]
+
 [dependencies]
 getrandom = { version = "0.2", features = ["js"] }
-substreams = "0.0.11"
-substreams-solana = "0.1.0"
+substreams = { workspace = true }
+substreams-solana = { workspace = true }
 ethabi = "17.2.0"
 hex-literal = "0.3.4"
-prost = { version = "0.10.1" }
-prost-types = "0.10.1"
-
-[lib]
-crate-type = ["cdylib"]
+prost = "0.11.2"
 
 [build-dependencies]
-prost-build = "0.10.1"
+anyhow = "1"
+substreams-common = { path = "../common" }

solana-sample/Makefile

@@ -4,4 +4,4 @@ build:
 
 .PHONY: run
 run:
-	substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml store_test -s 0 -t +10
+	substreams run substreams.yaml map_transfers -s 1000 -t +5 --debug-modules-initial-snapshot=store_pools

solana-sample/build.rs

@@ -0,0 +1,10 @@
+use anyhow::{Ok, Result};
+use substreams_common::codegen;
+
+fn main() -> Result<(), anyhow::Error> {
+    println!("cargo:rerun-if-changed=proto");
+    println!("cargo:rerun-if-changed=abi");
+    codegen::generate(None)?;
+
+    Ok(())
+}

solana-sample/proto/test.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+
+package messari.common.v1;
+
+message Block {
+  string item = 1;
+}
+

solana-sample/rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "1.64.0"
+components = [ "rustfmt" ]
+targets = [ "wasm32-unknown-unknown" ]

solana-sample/src/abi.rs

@@ -0,0 +1 @@
+// DO NOT EDIT - the file is generated by build script

solana-sample/src/lib.rs

@@ -1,7 +1,8 @@
-use substreams::{log, store};
+use substreams::pb;
+use substreams::store::{StoreSetRaw, StoreSet, StoreNew};
 use substreams_solana::pb::sol as solana;
 
 #[substreams::handlers::store]
-fn store_test(block: solana::v1::Block, _output: store::StoreSet) {
-    log::info!("block height: {}", block.blockhash);
+fn store_test(block: solana::v1::Block, _output: StoreSetRaw) {
+    _output.set(0, 0_u64, &block);
 }

solana-sample/src/pb.rs

@@ -0,0 +1,19 @@
+#[rustfmt::skip]
+#[path = "../target/pb/messari.common.v1.rs"]
+pub(in crate::pb) mod common_v1;
+
+pub mod common {
+    pub mod v1 {
+        pub use super::super::common_v1::*;
+    }
+}
+
+#[rustfmt::skip]
+#[path = "../target/pb/messari.solana.type.rs"]
+pub(in crate::pb) mod solana_type;
+
+pub mod solana {
+    pub mod type {
+        pub use super::super::solana_type::*;
+    }
+}

solana-sample/substreams.yaml

@@ -6,15 +6,22 @@ package:
 imports:
   sol: https://github.com/streamingfast/firehose-solana/releases/download/v0.1.0/solana-v0.1.0.spkg
 
+protobuf:
+  files:
+    - test.proto
+  importPaths:
+    - ./proto
+
 binaries:
   default:
     type: wasm/rust-v1
     file: "../target/wasm32-unknown-unknown/release/solana_sample.wasm"
 
 modules:
-    - name: store_test
-      kind: store
-      updatePolicy: set
-      valueType: bytes
-      inputs:
-        - source: sf.solana.type.v1.Block
+  # Will store the entire solana block as a string
+  - name: store_test
+    kind: store
+    updatePolicy: set
+    valueType: proto:sf.solana.type.v2.Block
+    inputs:
+      - source: sf.solana.type.v2.Block

spl-balance/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "spl-balance"
+version = "0.1.0"
+description = "Messari's standardized substream for solana SPL tokens"
+edition = "2021"
+repository = "https://github.com/messari/substreams"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+bs58 = "0.4.0"
+prost = "0.11.2"
+ethabi = "17.2.0"
+num-bigint = "0.4"
+hex-literal = "0.3.4"
+substreams = { workspace = true }
+substreams-solana = { workspace = true }
+substreams-helper = { path = "../substreams-helper" }
+
+[build-dependencies]
+anyhow = "1"
+substreams-common = { path = "../common" }

spl-balance/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build
+build:
+	cargo build --target wasm32-unknown-unknown --release
+
+.PHONY: run
+run:
+	substreams run -e mainnet.sol.streamingfast.io:443 substreams.yaml map_balances -s 173531340 -t +1

spl-balance/README.md

@@ -0,0 +1,7 @@
+# Solana SPL Token Balances
+
+This substream is designed to get the token balances of all SPL tokens on Solana. In addition `store_tokens` will get all SPL tokens and metadata.
+
+## Notes
+
+- TODO

spl-balance/build.rs

@@ -0,0 +1,10 @@
+use anyhow::{Ok, Result};
+use substreams_common::codegen;
+
+fn main() -> Result<(), anyhow::Error> {
+    println!("cargo:rerun-if-changed=proto");
+    println!("cargo:rerun-if-changed=abi");
+    codegen::generate(None)?;
+
+    Ok(())
+}

spl-balance/rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "1.64.0"
+components = [ "rustfmt" ]
+targets = [ "wasm32-unknown-unknown" ]

spl-balance/src/abi.rs

@@ -0,0 +1 @@
+// DO NOT EDIT - the file is generated by build script