diff --git a/cmd/lambda/main.go b/cmd/lambda/main.go index 223d66d..8306904 100644 --- a/cmd/lambda/main.go +++ b/cmd/lambda/main.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "os" "github.com/akrylysov/algnhsa" "github.com/aws/aws-sdk-go-v2/aws" @@ -13,6 +12,8 @@ import ( myconfig "github.com/nextdotid/proof_server/config" "github.com/nextdotid/proof_server/controller" "github.com/nextdotid/proof_server/model" + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" "github.com/nextdotid/proof_server/util/sqs" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" @@ -33,7 +34,8 @@ var ( ) func init_db(cfg aws.Config) { - model.Init(false) // TODO: should read auto migrate flag from ENV + shouldMigrate := util.GetE("DB_MIGRATE", "false") + model.Init(shouldMigrate == "true") } func init_sqs(cfg aws.Config) { @@ -70,6 +72,7 @@ func init() { init_db(cfg) init_sqs(cfg) init_validators() + // utilS3.Init(cfg) controller.Init() } @@ -81,8 +84,8 @@ func init_config_from_aws_secret() { if initialized { return } - secret_name := getE("SECRET_NAME", "") - region := getE("SECRET_REGION", "") + secret_name := util.GetE("SECRET_NAME", "") + region := util.GetE("SECRET_REGION", "") // Create a Secrets Manager client cfg, err := config.LoadDefaultConfig( @@ -116,18 +119,3 @@ func init_config_from_aws_secret() { } initialized = true } - -func getE(env_key, default_value string) string { - result := os.Getenv(env_key) - if len(result) == 0 { - if len(default_value) > 0 { - return default_value - } else { - logrus.Fatalf("ENV %s must be given! Abort.", env_key) - return "" - } - - } else { - return result - } -} diff --git a/cmd/lambda_worker/main.go b/cmd/lambda_worker/main.go index a8d52ec..b3aa680 100644 --- a/cmd/lambda_worker/main.go +++ b/cmd/lambda_worker/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strconv" "github.com/aws/aws-lambda-go/events" @@ -20,6 +19,8 @@ import ( myconfig "github.com/nextdotid/proof_server/config" "github.com/nextdotid/proof_server/model" "github.com/nextdotid/proof_server/types" + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" "github.com/nextdotid/proof_server/validator/discord" @@ -38,6 +39,7 @@ import ( ) var ( + awsConfig aws.Config initialized = false wallet *goar.Wallet ) @@ -74,11 +76,19 @@ func handler(ctx context.Context, sqs_event events.SQSEvent) (events.SQSEventRes case types.QueueActions.ArweaveUpload: arweaveMsgs[message.Persona] = raw_message.MessageId case types.QueueActions.Revalidate: - if err := revalidate_single(ctx, &message); err != nil { + if err := revalidateSingle(ctx, &message); err != nil { fmt.Printf("error revalidating proof record %d: %s\n", message.ProofID, err) // Ignore failed revalidation job since failed job will still update DB. // failures = append(failures, events.SQSBatchItemFailure{ItemIdentifier: raw_message.MessageId}) } + case types.QueueActions.TwitterOAuthTokenAcquire: + { + err := twitterRefreshOAuthToken() + if err != nil { + // Ignore errors for now + fmt.Printf("Error when retrieving Twitter OAuth key: %s", err.Error()) + } + } default: logrus.Warnf("unsupported queue action: %s", message.Action) failures = append(failures, events.SQSBatchItemFailure{ItemIdentifier: raw_message.MessageId}) @@ -126,7 +136,7 @@ func arweave_upload_many(personas []string) error { for _, pc := range chains { if pc.ArweaveID != "" { - continue + } previous, ok := lo.Find(chains, func(item *model.ProofChain) bool { @@ -137,7 +147,7 @@ func arweave_upload_many(personas []string) error { break } - item, err := arweave_bundle_single(pc, previous) + item, err := arweaveBundleSingle(pc, previous) if err != nil { logrus.Errorf("error marshalling proof chain %s: %w", pc.Uuid, err) break @@ -169,7 +179,7 @@ func arweave_upload_many(personas []string) error { return nil } -func arweave_bundle_single(pc *model.ProofChain, previous *model.ProofChain) (*artypes.BundleItem, error) { +func arweaveBundleSingle(pc *model.ProofChain, previous *model.ProofChain) (*artypes.BundleItem, error) { previousUuid := "" previousArweaveID := "" if previous != nil { @@ -224,7 +234,7 @@ func arweave_bundle_single(pc *model.ProofChain, previous *model.ProofChain) (*a return &item, nil } -func revalidate_single(ctx context.Context, message *types.QueueMessage) error { +func revalidateSingle(ctx context.Context, message *types.QueueMessage) error { proof := model.Proof{} tx := model.DB.Preload("ProofChain").Preload("ProofChain.Previous").Where("id = ?", message.ProofID).First(&proof) if tx.Error != nil { @@ -233,15 +243,16 @@ func revalidate_single(ctx context.Context, message *types.QueueMessage) error { return proof.Revalidate() } -func init_db(cfg aws.Config) { - model.Init(false) // TODO: should read auto migrate from ENV +func initDB() { + shouldMigrate := util.GetE("DB_MIGRATE", "false") + model.Init(shouldMigrate == "true") } // func init_sqs(cfg aws.Config) { // sqs.Init(cfg) // } -func init_validators() { +func initValidators() { twitter.Init() ethereum.Init() keybase.Init() @@ -256,7 +267,8 @@ func init_validators() { } func init() { - cfg, err := config.LoadDefaultConfig( + var err error + awsConfig, err = config.LoadDefaultConfig( context.Background(), config.WithRegion("ap-east-1"), ) @@ -264,20 +276,20 @@ func init() { logrus.Fatalf("Unable to load AWS config: %s", err) } common.CurrentRuntime = common.Runtimes.Lambda - init_config_from_aws_secret() + initConfigFromAWSSecret() logrus.SetLevel(logrus.InfoLevel) - init_db(cfg) + initDB() // init_sqs(cfg) - init_validators() + initValidators() } -func init_config_from_aws_secret() { +func initConfigFromAWSSecret() { if initialized { return } - secret_name := getE("SECRET_NAME", "") - region := getE("SECRET_REGION", "") + secretName := util.GetE("SECRET_NAME", "") + region := util.GetE("SECRET_REGION", "") // Create a Secrets Manager client cfg, err := config.LoadDefaultConfig( @@ -290,7 +302,7 @@ func init_config_from_aws_secret() { client := secretsmanager.NewFromConfig(cfg) input := secretsmanager.GetSecretValueInput{ - SecretId: aws.String(secret_name), + SecretId: aws.String(secretName), VersionStage: aws.String("AWSCURRENT"), } result, err := client.GetSecretValue(context.Background(), &input) @@ -319,17 +331,34 @@ func init_config_from_aws_secret() { initialized = true } -func getE(env_key, default_value string) string { - result := os.Getenv(env_key) - if len(result) == 0 { - if len(default_value) > 0 { - return default_value - } else { - logrus.Fatalf("ENV %s must be given! Abort.", env_key) - return "" - } +func twitterRefreshOAuthToken() (err error) { + const VALID_TOKEN_AMOUNT = 5 + if err = utilS3.Init(awsConfig); err != nil { + logrus.Fatalf("Error during initializing S3 client: %s", err.Error()) + } + ctx := context.Background() + tokens, err := twitter.GetTokenListFromS3(ctx) + if err != nil { + logrus.Fatalf("Error when loading Twitter token list from S3: %s", err.Error()) + } - } else { - return result + validTokens := lo.Filter(tokens.Tokens, func(token twitter.Token, _index int) bool { + return !token.IsExpired() + }) + if len(validTokens) < VALID_TOKEN_AMOUNT { + // Generate a new one + newToken, err := twitter.GenerateOauthToken() + if err != nil { + return err + } + fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens) + validTokens = append(validTokens, *newToken) + newTokenList := twitter.TokenList{ + Tokens: validTokens, + } + newTokenListJSON, _ := newTokenList.ToJSON() + utilS3.PutToS3(ctx, twitter.TWITTER_TOKEN_LIST_FILENAME, newTokenListJSON) } + + return nil } diff --git a/config/main.go b/config/main.go index 25c74d8..f2d8a4e 100644 --- a/config/main.go +++ b/config/main.go @@ -33,12 +33,18 @@ type HeadlessConfig struct { } type PlatformConfig struct { + Twitter TwitterPlatformConfig `json:"twitter"` Telegram TelegramPlatformConfig `json:"telegram"` Ethereum EthereumPlatformConfig `json:"ethereum"` Discord DiscordPlatformConfig `json:"discord"` Slack SlackPlatformConfig `json:"slack"` } +type TwitterPlatformConfig struct { + // Twitter API v2 Bearer token + OauthToken string `json:"oauth_token"` +} + type ArweaveConfig struct { Jwk string `json:"jwk"` ClientUrl string `json:"client_url"` diff --git a/go.mod b/go.mod index 10e6e40..21f69a0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/aws/aws-lambda-go v1.31.1 - github.com/aws/aws-sdk-go-v2 v1.16.5 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.15.4 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 github.com/everFinance/goar v1.4.2 @@ -26,15 +26,20 @@ require ( contrib.go.opencensus.io/exporter/stackdriver v0.13.4 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 // indirect - github.com/aws/smithy-go v1.11.3 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/blendle/zapdriver v1.3.1 // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect @@ -48,6 +53,7 @@ require ( github.com/everFinance/ttcrsa v1.1.3 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.5.3 // indirect + github.com/g8rswimmer/go-twitter/v2 v2.1.5 // indirect github.com/gagliardetto/binary v0.6.1 // indirect github.com/gagliardetto/treeout v0.1.4 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -148,6 +154,7 @@ require ( require ( github.com/akrylysov/algnhsa v0.12.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 github.com/bwmarrin/discordgo v0.25.0 github.com/ethereum/go-ethereum v1.10.25 diff --git a/go.sum b/go.sum index d9fe55d..6fa4dce 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,11 @@ github.com/aws/aws-lambda-go v1.31.1/go.mod h1:IF5Q7wj4VyZyUFnZ54IQqeWtctHQ9tz+K github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= github.com/aws/aws-sdk-go-v2/config v1.15.4 h1:P4mesY1hYUxru4f9SU0XxNKXmzfxsD0FtMIPRBjkH7Q= github.com/aws/aws-sdk-go-v2/config v1.15.4/go.mod h1:ZijHHh0xd/A+ZY53az0qzC5tT46kt4JVCePf2NX9Lk4= github.com/aws/aws-sdk-go-v2/credentials v1.12.0 h1:4R/NqlcRFSkR0wxOhgHi+agGpbEr5qMCjn7VqUIJY+E= @@ -84,15 +87,28 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.0/go.mod h1:9YWk7VW+eyKsoIL6/Cljk github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 h1:FP8gquGeGHHdfY6G5llaMQDF+HAf20VKc8opRwmjf04= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4/go.mod h1:u/s5/Z+ohUQOPXl00m2yJVyioWDECsbpXTQlaqSlufc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 h1:6cZRymlLEIlDTEB0+5+An6Zj1CKt6rSE69tOmFeu1nk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11/go.mod h1:0MR+sS1b/yxsfAPvAESrw8NfwUoxMinDyw6EYR9BS2U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 h1:b16QW0XWl0jWjLABFc1A+uh145Oqv+xDcObNk0iQgUk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 h1:m+mxqLIrGq7GJo5qw4rHn8BbUqHrvxvwFx54N1Pglvw= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6/go.mod h1:Z+i6uqZgCOBXhNoEGoRm/ZaLsaJA9rGUAmkVKM/3+g4= github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 h1:HlEYt9p1TAQYxeB8jz3y4dmXmZevX+cJnh8OU6x0aqo= @@ -102,8 +118,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.4/go.mod h1:cPDwJwsP4Kff9mldCXAmd github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 h1:+xtV90n3abQmgzk1pS++FdxZTrPEDgQng6e4/56WR2A= github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -206,6 +223,8 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.3 h1:vNFpj2z7YIbwh2bw7x35sqYpp2wfuq+pivKbWG09B8c= github.com/fsnotify/fsnotify v1.5.3/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/g8rswimmer/go-twitter/v2 v2.1.5 h1:Uj9Yuof2UducrP4Xva7irnUJfB9354/VyUXKmc2D5gg= +github.com/g8rswimmer/go-twitter/v2 v2.1.5/go.mod h1:/55xWb313KQs25X7oZrNSEwLQNkYHhPsDwFstc45vhc= github.com/gagliardetto/binary v0.6.1 h1:vGrbUym10xaaswadfnuSDr0xlP3NZS5XWbLqENJidrI= github.com/gagliardetto/binary v0.6.1/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0= github.com/gagliardetto/gofuzz v1.2.2/go.mod h1:bkH/3hYLZrMLbfYWA0pWzXmi5TTRZnu4pMGZBkqMKvY= diff --git a/types/mq.go b/types/mq.go index 0c6b8b3..bbd5bad 100644 --- a/types/mq.go +++ b/types/mq.go @@ -3,11 +3,13 @@ package types type QueueAction string var QueueActions = struct { - Revalidate QueueAction - ArweaveUpload QueueAction + Revalidate QueueAction + ArweaveUpload QueueAction + TwitterOAuthTokenAcquire QueueAction }{ - Revalidate: "revalidate", - ArweaveUpload: "arweave_upload", + Revalidate: "revalidate", + ArweaveUpload: "arweave_upload", + TwitterOAuthTokenAcquire: "twitter_oauth_token_acquire", } // QueueMessage indicates structure of messages in Amazon SQS. diff --git a/util/s3/main.go b/util/s3/main.go new file mode 100644 index 0000000..a100f9a --- /dev/null +++ b/util/s3/main.go @@ -0,0 +1,47 @@ +package s3 + +import ( + "bytes" + "context" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/nextdotid/proof_server/util" +) + +var ( + s3Client *s3.Client + bucketName string +) + +func Init(awsConfig aws.Config) error { + if s3Client == nil { + s3Client = s3.NewFromConfig(awsConfig) + } + bucketName = util.GetE("S3_BUCKET", "") + return nil +} + +// ReadFromS3 should be called before Init() +func ReadFromS3(ctx context.Context, key string) (content []byte, err error) { + result, err := s3Client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err != nil { + return []byte{}, err + } + defer result.Body.Close() + return io.ReadAll(result.Body) +} + +// PutToS3 should be called before Init() +func PutToS3(ctx context.Context, key string, content []byte) (err error) { + _, err = s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: bytes.NewReader(content), + }) + return err +} diff --git a/util/util.go b/util/util.go index 6a06e8e..cc5f5bc 100644 --- a/util/util.go +++ b/util/util.go @@ -2,10 +2,12 @@ package util import ( "encoding/base64" + "os" "strconv" "time" "github.com/nextdotid/proof_server/util/base1024" + "github.com/sirupsen/logrus" "golang.org/x/xerrors" ) @@ -30,3 +32,19 @@ func DecodeString(s string) ([]byte, error) { } return base1024.DecodeString(s) } + +// GetE gets current system's environment variable as `string`. +// If `defaultValue` is empty, this environment key must be exist, or it will panic. +func GetE(envKey, defaultValue string) string { + result := os.Getenv(envKey) + if len(result) == 0 { + if len(defaultValue) > 0 { + return defaultValue + } else { + logrus.Fatalf("ENV %s must be given! Abort.", envKey) + return "" + } + + } + return result +} diff --git a/validator/twitter/api.go b/validator/twitter/api.go new file mode 100644 index 0000000..d53d110 --- /dev/null +++ b/validator/twitter/api.go @@ -0,0 +1,95 @@ +package twitter + +import ( + "context" + "fmt" + "net/http" + "strings" + + twitter "github.com/g8rswimmer/go-twitter/v2" + "github.com/nextdotid/proof_server/config" + "golang.org/x/xerrors" +) + +type APIResponse struct { + User struct { + ID string `json:"user_id"` + ScreenName string `json:"screen_name"` + } `json:"user"` + Text string `json:"text"` +} + +var ( + twitterClient *twitter.Client + CurrentTokenList *TokenList +) + +type authorize struct { + Token string +} + +func (a authorize) Add(req *http.Request) { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", a.Token)) +} + +// Fetch tweet using twitter OAuth2.0 API. +// FIXME: should be switched to guest OAuth token solution. +func fetchPostWithAPI(id string, maxRetries int) (*APIResponse, error) { + if twitterClient == nil { + twitterClient = &twitter.Client{ + Authorizer: authorize{ + Token: config.C.Platform.Twitter.OauthToken, + }, + Client: http.DefaultClient, + Host: "https://api.twitter.com", + } + } + opts := twitter.TweetLookupOpts{ + Expansions: []twitter.Expansion{twitter.ExpansionEntitiesMentionsUserName, twitter.ExpansionAuthorID}, + TweetFields: []twitter.TweetField{twitter.TweetFieldText, twitter.TweetFieldCreatedAt, twitter.TweetFieldEntities}, + } + result, err := twitterClient.TweetLookup(context.Background(), []string{id}, opts) + if err != nil { + return nil, xerrors.Errorf("error when retriving tweet: %w", err) + } + tweet := result.Raw.Tweets[0] + if tweet == nil { + return nil, xerrors.Errorf("tweet not found: %s", id) + } + + response := APIResponse{ + Text: tweet.Text, + } + response.User.ID = tweet.AuthorID + if len(tweet.Entities.Mentions) > 0 { + // Expect to be the user himself + mention := tweet.Entities.Mentions[0] + response.User.ScreenName = strings.ToLower(mention.UserName) + } + + return &response, nil +} + +// func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { +// const RETRY_AFTER = time.Second +// ctx := context.Background() +// if CurrentTokenList == nil { +// CurrentTokenList, err = GetTokenListFromS3(ctx) +// if err != nil { +// return nil, xerrors.Errorf("fetchPostWithAPI: %w", err) +// } +// if CurrentTokenList == nil { +// return nil, xerrors.Errorf("twitter token list does not exist") +// } +// } +// token := lo.Sample(CurrentTokenList.Tokens) +// if lo.IsEmpty(token.OAuthSecret) || lo.IsEmpty(token.OAuthKey) { +// return nil, xerrors.Errorf("twitter token seems to be empty") +// } + +// // TODO: Use token to query specific tweet with twitter API +// // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets +// // https://api.twitter.com/1.1/statuses/show.json + +// return nil, nil +// } diff --git a/validator/twitter/api_test.go b/validator/twitter/api_test.go new file mode 100644 index 0000000..f39f2cd --- /dev/null +++ b/validator/twitter/api_test.go @@ -0,0 +1,17 @@ +package twitter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fetchPostWithAPI(t *testing.T) { + t.Run("success", func(t *testing.T) { + tweet, err := fetchPostWithAPI("1652176440396517378", 10) + require.NoError(t, err) + require.Contains(t, tweet.Text, "Sig:") + require.Equal(t, tweet.User.ScreenName, "bgm38") + require.Equal(t, tweet.User.ID, "292254624") + }) +} diff --git a/validator/twitter/syndication_api.go b/validator/twitter/syndication_api.go deleted file mode 100644 index 2ba48ca..0000000 --- a/validator/twitter/syndication_api.go +++ /dev/null @@ -1,174 +0,0 @@ -package twitter - -import ( - "encoding/json" - "io" - "net/http" - "time" - - "github.com/samber/lo" - "golang.org/x/xerrors" -) - -type SyndicationAPIResponse struct { - ID string `json:"id_str"` - User SyndicationAPIUser `json:"user"` - Text string `json:"text"` -} - -type SyndicationAPIUser struct { - ID string `json:"id_str"` - ScreenName string `json:"screen_name"` -} - -// / data.threaded_conversation_with_injections.instructions[0].entries[?].content.itemContent.tweet_results.result.legacy.full_text -// / data.threaded_conversation_with_injections.instructions[0].entries[?].content.itemContent.tweet_results.result.core.user_results.result.legacy.name -// / ?: find entryId = "tweet-TWEETID" -type GraphQLAPIResponse struct { - Data struct { - ThreadedConversationWithInjections struct { - Instructions []struct { - Entries []GraphQLAPIEntry `json:"entries"` - } `json:"instructions"` - } `json:"threaded_conversation_with_injections"` - } `json:"data"` -} - -type GraphQLAPIEntry struct { - EntryID string `json:"entryId"` - Content struct { - ItemContent struct { - TweetResults struct { - Result struct { - Core struct { - UserResults struct { - Result struct { - RestID string `json:"rest_id"` - Legacy struct { - ScreenName string `json:"screen_name"` - } `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - Legacy struct { - FullText string `json:"full_text"` - } `json:"legacy"` - } `json:"result"` - } `json:"tweet_results"` - } `json:"itemContent"` - } `json:"content"` -} - -const ( - GUEST_TOKEN_REQUEST = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" - - QUERY_URL_HEAD = "https://api.twitter.com/graphql/miKSMGb2R1SewIJv2-ablQ/TweetDetail?variables=%7B%22focalTweetId%22%3A%22" - QUERY_URL_TAIL = "%22,%22withBirdwatchNotes%22%3Afalse,%22includePromotedContent%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withVoice%22%3Afalse,%22withV2Timeline%22%3Afalse%7D&features=%7B%22blue_business_profile_image_shape_enabled%22%3Afalse,%22rweb_lists_timeline_redesign_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue,%22responsive_web_graphql_timeline_navigation_enabled%22%3Afalse,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Afalse,%22vibe_api_enabled%22%3Afalse,%22responsive_web_edit_tweet_api_enabled%22%3Afalse,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Afalse,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Afalse,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Afalse,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_rich_text_read_enabled%22%3Afalse,%22longform_notetweets_inline_media_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D" - USER_AGENT = "User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -) - -// something like "1691388211879432192" -var GuestToken string - -func fetchPostWithSyndication(id string, maxRetries int) (tweet *SyndicationAPIResponse, err error) { - const RETRY_AFTER = time.Second - - accumulatedErrors := "" - for retry := 0; retry < maxRetries; retry++ { - if retry != 0 { - time.Sleep(RETRY_AFTER) - } - // Fetching guestToken - err := fetchGuestToken() - if err != nil { - accumulatedErrors += (err.Error() + "; ") - continue - } - - tweet, err := fetchPost(id) - if err != nil { - accumulatedErrors += (err.Error() + "; ") - continue - } - return tweet, nil - } - return nil, xerrors.Errorf("%d retries reached: %s", maxRetries, accumulatedErrors) -} - -func fetchGuestToken() (err error) { - if GuestToken != "" { - return nil - } - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) - if err != nil { - return err - } - req.Header.Set("Authorization", GUEST_TOKEN_REQUEST) - req.Header.Set("User-Agent", USER_AGENT) - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - var guestTokenResponse struct { - GuestToken string `json:"guest_token"` - } - err = json.Unmarshal(body, &guestTokenResponse) - if err != nil { - return err - } - if guestTokenResponse.GuestToken == "" { - return xerrors.Errorf("Guest token is empty") - } - - GuestToken = guestTokenResponse.GuestToken - - return nil -} - -func fetchPost(postID string) (post *SyndicationAPIResponse, err error) { - req, err := http.NewRequest("GET", QUERY_URL_HEAD+postID+QUERY_URL_TAIL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", GUEST_TOKEN_REQUEST) - req.Header.Set("User-Agent", USER_AGENT) - req.Header.Set("x-guest-token", GuestToken) - - resp, err := new(http.Client).Do(req) - if err != nil { - return nil, err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - response := new(GraphQLAPIResponse) - err = json.Unmarshal(body, response) - if err != nil { - return nil, err - } - if len(response.Data.ThreadedConversationWithInjections.Instructions) == 0 { - return nil, xerrors.Errorf("No instructions found in response") - } - instruction := response.Data.ThreadedConversationWithInjections.Instructions[0] - entry, found := lo.Find(instruction.Entries, func(entry GraphQLAPIEntry) bool { - return entry.EntryID == ("tweet-" + postID) - }) - if !found { - return nil, xerrors.Errorf("Tweet specified in ProofLocation is not found in API response") - } - - return &SyndicationAPIResponse{ - ID: postID, - User: SyndicationAPIUser{ - ID: entry.Content.ItemContent.TweetResults.Result.Core.UserResults.Result.RestID, - ScreenName: entry.Content.ItemContent.TweetResults.Result.Core.UserResults.Result.Legacy.ScreenName, - }, - Text: entry.Content.ItemContent.TweetResults.Result.Legacy.FullText, - }, nil -} diff --git a/validator/twitter/syndication_api_test.go b/validator/twitter/syndication_api_test.go deleted file mode 100644 index 247c8da..0000000 --- a/validator/twitter/syndication_api_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package twitter - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_SyndicationAPI(t *testing.T) { - t.Run("success", func(t *testing.T) { - // Read /tmp/result_without_timeline.json file and deserialize it into GraphQLResponse - // Then compare it with expected GraphQLResponse - postID := "1687007065032814593" - result, err := fetchPostWithSyndication(postID, 1) - require.NoError(t, err) - require.Equal(t, postID, result.ID) - require.Equal(t, "bgm38", result.User.ScreenName) - require.Equal(t, "292254624", result.User.ID) - }) -} diff --git a/validator/twitter/token.go b/validator/twitter/token.go new file mode 100644 index 0000000..a798dd8 --- /dev/null +++ b/validator/twitter/token.go @@ -0,0 +1,418 @@ +package twitter + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" + "github.com/samber/lo" + "golang.org/x/xerrors" +) + +const ( + TWITTER_TOKEN_LIST_FILENAME string = "config/proof_service/twitter_oauth_tokens.json" +) + +type Token struct { + AccessToken string `json:"access_token"` + GuestToken string `json:"guest_token"` + FlowToken string `json:"flow_token"` + OAuthKey string `json:"oauth_key"` + OAuthSecret string `json:"oauth_secret"` + CreatedAt string `json:"created_at"` +} + +type TokenList struct { + Tokens []Token `json:"tokens"` +} + +const ( + TWITTER_CONSUMER_KEY = "3rJOl1ODzm9yZy63FACdg" + TWITTER_CONSUMER_SECRET = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" +) + +func setHeaders(req *http.Request, tokens *Token, setAccessToken, setGuestToken bool) { + req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Twitter-API-Version", "5") + req.Header.Set("X-Twitter-Client", "TwitterAndroid") + req.Header.Set("X-Twitter-Client-Version", "9.95.0-release.0") + req.Header.Set("OS-Version", "28") + req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") + req.Header.Set("X-Twitter-Active-User", "yes") + if setGuestToken { + req.Header.Set("X-Guest-Token", tokens.GuestToken) + } + if setAccessToken { + req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) + } +} + +// GenerateOauthToken generates a new Twitter OAuth guest token +// which can be used in calling Official APIs. +func GenerateOauthToken() (token *Token, err error) { + token = new(Token) + token.CreatedAt = util.TimeToTimestampString(time.Now()) + + if err := token.getFlowToken(); err != nil { + return nil, err + } + l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", token.AccessToken, token.GuestToken, token.FlowToken) + + requestBody := fmt.Sprintf(`{ + "flow_token": "%s", + "subtask_inputs": [ + { + "open_link": { + "link": "next_link" + }, + "subtask_id": "NextTaskOpenLink" + } + ], + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }`, token.FlowToken) + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) + if err != nil { + return nil, err + } + setHeaders(req, token, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return nil, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + l.Infof("Response: \n%s\n", body) + type ResponseSubtask struct { + // Should exist + OpenAccount *struct { + OauthToken string `json:"oauth_token"` + OauthTokenSecret string `json:"oauth_token_secret"` + } `json:"open_account"` + } + + type Response struct { + // Should be empty + Errors *[]struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"errors"` + // Should be "success" + Status string `json:"status"` + // A new flow token, usually ends with ":3" + FlowToken string `json:"flow_token"` + Subtasks []ResponseSubtask `json:"subtasks"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return nil, err + } + if response.Errors != nil { + return nil, xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) + } + if response.Status != "success" { + return nil, xerrors.Errorf("wrong API status: %s", response.Status) + } + + st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { + return (subtask.OpenAccount != nil) + }) + if !found { + return nil, xerrors.Errorf("oauth token not found in response") + } + // Update new FlowToken + token.FlowToken = response.FlowToken + l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) + + return token, nil +} + +func (token *Token) getFlowToken() (err error) { + if token.GuestToken == "" { + if err := token.getGuestToken(); err != nil { + return err + } + } + + requestBody := `{ + "flow_token": null, + "input_flow_data": { + "country_code": null, + "flow_context": { + "start_location": { + "location": "splash_screen" + } + }, + "requested_variant": null, + "target_user_id": 0 + }, + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }` + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json?flow_name=welcome&api_version=1&known_device_token=&sim_country_code=us", strings.NewReader(requestBody)) + if err != nil { + return err + } + setHeaders(req, token, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + type Response struct { + FlowToken string `json:"flow_token"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.FlowToken == "" { + return xerrors.Errorf("empty FlowToken") + } + + token.FlowToken = response.FlowToken + return nil +} + +func (token *Token) getGuestToken() (err error) { + if token.GuestToken != "" { + return nil + } + if token.AccessToken == "" { + if err = token.getAccessToken(); err != nil { + return err + } + } + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) + if err != nil { + return err + } + setHeaders(req, token, true, false) + type Response struct { + GuestToken string `json:"guest_token"` + } + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + if response.GuestToken == "" { + return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) + } + token.GuestToken = response.GuestToken + + return nil +} + +func (token *Token) getAccessToken() (err error) { + if token.AccessToken != "" { + return nil + } + + type Response struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + } + req, err := http.NewRequest("POST", "https://api.twitter.com/oauth2/token?grant_type=client_credentials", nil) + if err != nil { + return err + } + setHeaders(req, token, false, false) + req.SetBasicAuth(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET) + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.TokenType != "bearer" || len(response.AccessToken) == 0 { + return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) + } + + token.AccessToken = response.AccessToken + return nil +} + +func (token *Token) IsExpired() bool { + const EXPIRED_AT = "24h" + expiredAt, _ := time.ParseDuration(EXPIRED_AT) + createdAt, _ := util.TimestampStringToTime(token.CreatedAt) + return createdAt.Add(expiredAt).Before(time.Now()) +} + +func (tl *TokenList) ToJSON() ([]byte, error) { + return json.Marshal(*tl) +} + +func TokenListFromJSON(content []byte) (*TokenList, error) { + tl := new(TokenList) + err := json.Unmarshal(content, tl) + if err != nil { + return nil, err + } + return tl, nil +} + +// GetTokenListFromS3 reads JSON file in AWS S3 to get token lists. +func GetTokenListFromS3(ctx context.Context) (*TokenList, error) { + body, _ := utilS3.ReadFromS3(ctx, TWITTER_TOKEN_LIST_FILENAME) + // If file not found, touch a new one: + if len(body) == 0 { + tl := new(TokenList) + newTokenList, _ := tl.ToJSON() + if err := utilS3.PutToS3(ctx, TWITTER_TOKEN_LIST_FILENAME, newTokenList); err != nil { + return nil, err + } + return tl, nil + } + + // File found, read it + return TokenListFromJSON(body) +} diff --git a/validator/twitter/token_test.go b/validator/twitter/token_test.go new file mode 100644 index 0000000..d671f37 --- /dev/null +++ b/validator/twitter/token_test.go @@ -0,0 +1,45 @@ +package twitter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_getAccessToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + tokens := new(Token) + require.NoError(t, tokens.getAccessToken()) + require.NotEmpty(t, tokens.AccessToken) + t.Logf("Access token: %s", tokens.AccessToken) + }) +} + +func Test_getGuestToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + tokens := new(Token) + require.NoError(t, tokens.getGuestToken()) + require.NotEmpty(t, tokens.GuestToken) + t.Logf("Guest token: %s", tokens.GuestToken) + }) +} + +func Test_FlowToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + tokens := new(Token) + require.NoError(t, tokens.getFlowToken()) + require.NotEmpty(t, tokens.FlowToken) + t.Logf("Flow token: %s", tokens.FlowToken) + }) +} + +func Test_OauthToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + tokens, err := GenerateOauthToken() + require.NoError(t, err) + require.False(t, tokens.IsExpired()) + t.Logf("Access token: %s", tokens.AccessToken) + t.Logf("Guest token: %s", tokens.GuestToken) + t.Logf("Flow token: %s", tokens.FlowToken) + }) +} diff --git a/validator/twitter/twitter.go b/validator/twitter/twitter.go index ab935cc..fb302c4 100644 --- a/validator/twitter/twitter.go +++ b/validator/twitter/twitter.go @@ -104,7 +104,7 @@ func (twitter *Twitter) Validate() (err error) { // return xerrors.Errorf("fetching tweet with headless browser: %w", err) // } - tweet, err := fetchPostWithSyndication(fmt.Sprint(tweetID), 3) + tweet, err := fetchPostWithAPI(fmt.Sprint(tweetID), 3) if err != nil { return xerrors.Errorf("fetching tweet with syndication API: %w", err) }