From c83e401f1ab49d018d2636f33e2b188c5fea9a46 Mon Sep 17 00:00:00 2001 From: Nyk Ma Date: Sun, 17 Sep 2023 00:26:36 +0800 Subject: [PATCH 1/5] [#] validator/twitter: OAuth guest token acquire --- validator/twitter/api.go | 378 ++++++++++++++++++++++ validator/twitter/api_test.go | 40 +++ validator/twitter/syndication_api.go | 174 ---------- validator/twitter/syndication_api_test.go | 20 -- validator/twitter/twitter.go | 2 +- 5 files changed, 419 insertions(+), 195 deletions(-) create mode 100644 validator/twitter/api.go create mode 100644 validator/twitter/api_test.go delete mode 100644 validator/twitter/syndication_api.go delete mode 100644 validator/twitter/syndication_api_test.go diff --git a/validator/twitter/api.go b/validator/twitter/api.go new file mode 100644 index 0000000..ee88261 --- /dev/null +++ b/validator/twitter/api.go @@ -0,0 +1,378 @@ +package twitter + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/samber/lo" + "golang.org/x/xerrors" +) + +type APIResponse struct { + User struct { + ID string `json:"user_id"` + ScreenName string `json:"screen_name"` + } `json:"user"` + Text string `json:"text"` +} + +const ( + BASIC_AUTH_USERNAME = "3rJOl1ODzm9yZy63FACdg" + BASIC_AUTH_PASSWORD = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" +) + +var ( + // TODO: should save accessToken to somewhere else (shared by all Lambda instances) + accessToken string + guestToken string + flowToken string +) + +func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { + const RETRY_AFTER = time.Second + + return nil, nil +} + +func setHeaders(req *http.Request, setAccessToken, setGuestToken bool) { + req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Twitter-API-Version", "5") + req.Header.Set("X-Twitter-Client", "TwitterAndroid") + req.Header.Set("X-Twitter-Client-Version", "9.95.0-release.0") + req.Header.Set("OS-Version", "28") + req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") + req.Header.Set("X-Twitter-Active-User", "yes") + if setGuestToken { + req.Header.Set("X-Guest-Token", guestToken) + } + if setAccessToken { + req.Header.Set("Authorization", "Bearer "+accessToken) + } +} + +func GetOauthToken() (err error) { + if flowToken == "" { + if err := getFlowToken(); err != nil { + return err + } + } + l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", accessToken, guestToken, flowToken) + + requestBody := fmt.Sprintf(`{ + "flow_token": "%s", + "subtask_inputs": [ + { + "open_link": { + "link": "next_link" + }, + "subtask_id": "NextTaskOpenLink" + } + ], + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }`, flowToken) + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) + if err != nil { + return err + } + setHeaders(req, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + l.Infof("Response: \n%s\n", body) + type ResponseSubtask struct { + // Should exist + OpenAccount *struct { + OauthToken string `json:"oauth_token"` + OauthTokenSecret string `json:"oauth_token_secret"` + } `json:"open_account"` + } + + type Response struct { + // Should be empty + Errors *[]struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"errors"` + // Should be "success" + Status string `json:"status"` + // A new flow token, usually ends with ":3" + FlowToken string `json:"flow_token"` + Subtasks []ResponseSubtask `json:"subtasks"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + if response.Errors != nil { + return xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) + } + if response.Status != "success" { + return xerrors.Errorf("wrong API status: %s", response.Status) + } + + st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { + return (subtask.OpenAccount != nil) + }) + if !found { + return xerrors.Errorf("oauth token not found in response") + } + flowToken = response.FlowToken + l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) + + return nil +} + +func getFlowToken() (err error) { + if guestToken == "" { + if err := getGuestToken(); err != nil { + return err + } + } + + requestBody := `{ + "flow_token": null, + "input_flow_data": { + "country_code": null, + "flow_context": { + "start_location": { + "location": "splash_screen" + } + }, + "requested_variant": null, + "target_user_id": 0 + }, + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }` + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json?flow_name=welcome&api_version=1&known_device_token=&sim_country_code=us", strings.NewReader(requestBody)) + if err != nil { + return err + } + setHeaders(req, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + type Response struct { + FlowToken string `json:"flow_token"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.FlowToken == "" { + return xerrors.Errorf("empty FlowToken") + } + + flowToken = response.FlowToken + return nil +} + +func getGuestToken() (err error) { + if guestToken != "" { + return nil + } + if accessToken == "" { + if err = getAccessToken(); err != nil { + return err + } + } + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) + if err != nil { + return err + } + setHeaders(req, true, false) + type Response struct { + GuestToken string `json:"guest_token"` + } + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + if response.GuestToken == "" { + return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) + } + guestToken = response.GuestToken + + return nil +} + +func getAccessToken() (err error) { + if accessToken != "" { + return nil + } + + type Response struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + } + req, err := http.NewRequest("POST", "https://api.twitter.com/oauth2/token?grant_type=client_credentials", nil) + if err != nil { + return err + } + setHeaders(req, false, false) + req.SetBasicAuth(BASIC_AUTH_USERNAME, BASIC_AUTH_PASSWORD) + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.TokenType != "bearer" || len(response.AccessToken) == 0 { + return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) + } + + accessToken = response.AccessToken + return nil +} diff --git a/validator/twitter/api_test.go b/validator/twitter/api_test.go new file mode 100644 index 0000000..92b6108 --- /dev/null +++ b/validator/twitter/api_test.go @@ -0,0 +1,40 @@ +package twitter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_getAccessToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + require.NoError(t, getAccessToken()) + require.NotEmpty(t, accessToken) + t.Logf("Access token: %s", accessToken) + }) +} + +func Test_getGuestToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + require.NoError(t, getGuestToken()) + require.NotEmpty(t, guestToken) + t.Logf("Guest token: %s", guestToken) + }) +} + +func Test_FlowToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + require.NoError(t, getFlowToken()) + require.NotEmpty(t, flowToken) + t.Logf("Flow token: %s", flowToken) + }) +} + +func Test_OauthToken(t *testing.T) { + t.Run("success", func(t *testing.T) { + require.NoError(t, GetOauthToken()) + t.Logf("Access token: %s", accessToken) + t.Logf("Guest token: %s", guestToken) + t.Logf("Flow token: %s", flowToken) + }) +} diff --git a/validator/twitter/syndication_api.go b/validator/twitter/syndication_api.go deleted file mode 100644 index 2ba48ca..0000000 --- a/validator/twitter/syndication_api.go +++ /dev/null @@ -1,174 +0,0 @@ -package twitter - -import ( - "encoding/json" - "io" - "net/http" - "time" - - "github.com/samber/lo" - "golang.org/x/xerrors" -) - -type SyndicationAPIResponse struct { - ID string `json:"id_str"` - User SyndicationAPIUser `json:"user"` - Text string `json:"text"` -} - -type SyndicationAPIUser struct { - ID string `json:"id_str"` - ScreenName string `json:"screen_name"` -} - -// / data.threaded_conversation_with_injections.instructions[0].entries[?].content.itemContent.tweet_results.result.legacy.full_text -// / data.threaded_conversation_with_injections.instructions[0].entries[?].content.itemContent.tweet_results.result.core.user_results.result.legacy.name -// / ?: find entryId = "tweet-TWEETID" -type GraphQLAPIResponse struct { - Data struct { - ThreadedConversationWithInjections struct { - Instructions []struct { - Entries []GraphQLAPIEntry `json:"entries"` - } `json:"instructions"` - } `json:"threaded_conversation_with_injections"` - } `json:"data"` -} - -type GraphQLAPIEntry struct { - EntryID string `json:"entryId"` - Content struct { - ItemContent struct { - TweetResults struct { - Result struct { - Core struct { - UserResults struct { - Result struct { - RestID string `json:"rest_id"` - Legacy struct { - ScreenName string `json:"screen_name"` - } `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - Legacy struct { - FullText string `json:"full_text"` - } `json:"legacy"` - } `json:"result"` - } `json:"tweet_results"` - } `json:"itemContent"` - } `json:"content"` -} - -const ( - GUEST_TOKEN_REQUEST = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" - - QUERY_URL_HEAD = "https://api.twitter.com/graphql/miKSMGb2R1SewIJv2-ablQ/TweetDetail?variables=%7B%22focalTweetId%22%3A%22" - QUERY_URL_TAIL = "%22,%22withBirdwatchNotes%22%3Afalse,%22includePromotedContent%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withVoice%22%3Afalse,%22withV2Timeline%22%3Afalse%7D&features=%7B%22blue_business_profile_image_shape_enabled%22%3Afalse,%22rweb_lists_timeline_redesign_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue,%22responsive_web_graphql_timeline_navigation_enabled%22%3Afalse,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Afalse,%22vibe_api_enabled%22%3Afalse,%22responsive_web_edit_tweet_api_enabled%22%3Afalse,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Afalse,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Afalse,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Afalse,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_rich_text_read_enabled%22%3Afalse,%22longform_notetweets_inline_media_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D" - USER_AGENT = "User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -) - -// something like "1691388211879432192" -var GuestToken string - -func fetchPostWithSyndication(id string, maxRetries int) (tweet *SyndicationAPIResponse, err error) { - const RETRY_AFTER = time.Second - - accumulatedErrors := "" - for retry := 0; retry < maxRetries; retry++ { - if retry != 0 { - time.Sleep(RETRY_AFTER) - } - // Fetching guestToken - err := fetchGuestToken() - if err != nil { - accumulatedErrors += (err.Error() + "; ") - continue - } - - tweet, err := fetchPost(id) - if err != nil { - accumulatedErrors += (err.Error() + "; ") - continue - } - return tweet, nil - } - return nil, xerrors.Errorf("%d retries reached: %s", maxRetries, accumulatedErrors) -} - -func fetchGuestToken() (err error) { - if GuestToken != "" { - return nil - } - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) - if err != nil { - return err - } - req.Header.Set("Authorization", GUEST_TOKEN_REQUEST) - req.Header.Set("User-Agent", USER_AGENT) - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - var guestTokenResponse struct { - GuestToken string `json:"guest_token"` - } - err = json.Unmarshal(body, &guestTokenResponse) - if err != nil { - return err - } - if guestTokenResponse.GuestToken == "" { - return xerrors.Errorf("Guest token is empty") - } - - GuestToken = guestTokenResponse.GuestToken - - return nil -} - -func fetchPost(postID string) (post *SyndicationAPIResponse, err error) { - req, err := http.NewRequest("GET", QUERY_URL_HEAD+postID+QUERY_URL_TAIL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", GUEST_TOKEN_REQUEST) - req.Header.Set("User-Agent", USER_AGENT) - req.Header.Set("x-guest-token", GuestToken) - - resp, err := new(http.Client).Do(req) - if err != nil { - return nil, err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - response := new(GraphQLAPIResponse) - err = json.Unmarshal(body, response) - if err != nil { - return nil, err - } - if len(response.Data.ThreadedConversationWithInjections.Instructions) == 0 { - return nil, xerrors.Errorf("No instructions found in response") - } - instruction := response.Data.ThreadedConversationWithInjections.Instructions[0] - entry, found := lo.Find(instruction.Entries, func(entry GraphQLAPIEntry) bool { - return entry.EntryID == ("tweet-" + postID) - }) - if !found { - return nil, xerrors.Errorf("Tweet specified in ProofLocation is not found in API response") - } - - return &SyndicationAPIResponse{ - ID: postID, - User: SyndicationAPIUser{ - ID: entry.Content.ItemContent.TweetResults.Result.Core.UserResults.Result.RestID, - ScreenName: entry.Content.ItemContent.TweetResults.Result.Core.UserResults.Result.Legacy.ScreenName, - }, - Text: entry.Content.ItemContent.TweetResults.Result.Legacy.FullText, - }, nil -} diff --git a/validator/twitter/syndication_api_test.go b/validator/twitter/syndication_api_test.go deleted file mode 100644 index 247c8da..0000000 --- a/validator/twitter/syndication_api_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package twitter - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_SyndicationAPI(t *testing.T) { - t.Run("success", func(t *testing.T) { - // Read /tmp/result_without_timeline.json file and deserialize it into GraphQLResponse - // Then compare it with expected GraphQLResponse - postID := "1687007065032814593" - result, err := fetchPostWithSyndication(postID, 1) - require.NoError(t, err) - require.Equal(t, postID, result.ID) - require.Equal(t, "bgm38", result.User.ScreenName) - require.Equal(t, "292254624", result.User.ID) - }) -} diff --git a/validator/twitter/twitter.go b/validator/twitter/twitter.go index ab935cc..fb302c4 100644 --- a/validator/twitter/twitter.go +++ b/validator/twitter/twitter.go @@ -104,7 +104,7 @@ func (twitter *Twitter) Validate() (err error) { // return xerrors.Errorf("fetching tweet with headless browser: %w", err) // } - tweet, err := fetchPostWithSyndication(fmt.Sprint(tweetID), 3) + tweet, err := fetchPostWithAPI(fmt.Sprint(tweetID), 3) if err != nil { return xerrors.Errorf("fetching tweet with syndication API: %w", err) } From 3b0a26657431080de5ba9f4e3b844c426c279c56 Mon Sep 17 00:00:00 2001 From: Nyk Ma Date: Sun, 17 Sep 2023 01:17:34 +0800 Subject: [PATCH 2/5] [#] lambda_worker: Add twitter OAuth key retrieve job type --- cmd/lambda_worker/main.go | 58 +++++++++++++------ types/mq.go | 10 ++-- validator/twitter/api.go | 106 +++++++++++++++++++--------------- validator/twitter/api_test.go | 31 +++++----- 4 files changed, 125 insertions(+), 80 deletions(-) diff --git a/cmd/lambda_worker/main.go b/cmd/lambda_worker/main.go index a8d52ec..36027f3 100644 --- a/cmd/lambda_worker/main.go +++ b/cmd/lambda_worker/main.go @@ -74,11 +74,19 @@ func handler(ctx context.Context, sqs_event events.SQSEvent) (events.SQSEventRes case types.QueueActions.ArweaveUpload: arweaveMsgs[message.Persona] = raw_message.MessageId case types.QueueActions.Revalidate: - if err := revalidate_single(ctx, &message); err != nil { + if err := revalidateSingle(ctx, &message); err != nil { fmt.Printf("error revalidating proof record %d: %s\n", message.ProofID, err) // Ignore failed revalidation job since failed job will still update DB. // failures = append(failures, events.SQSBatchItemFailure{ItemIdentifier: raw_message.MessageId}) } + case types.QueueActions.TwitterOAuthTokenAcquire: + { + err := twitterRetrieveOAuthToken() + if err != nil { + // Ignore errors for now + fmt.Printf("Error when retrieving Twitter OAuth key: %s", err.Error()) + } + } default: logrus.Warnf("unsupported queue action: %s", message.Action) failures = append(failures, events.SQSBatchItemFailure{ItemIdentifier: raw_message.MessageId}) @@ -137,7 +145,7 @@ func arweave_upload_many(personas []string) error { break } - item, err := arweave_bundle_single(pc, previous) + item, err := arweaveBundleSingle(pc, previous) if err != nil { logrus.Errorf("error marshalling proof chain %s: %w", pc.Uuid, err) break @@ -169,7 +177,7 @@ func arweave_upload_many(personas []string) error { return nil } -func arweave_bundle_single(pc *model.ProofChain, previous *model.ProofChain) (*artypes.BundleItem, error) { +func arweaveBundleSingle(pc *model.ProofChain, previous *model.ProofChain) (*artypes.BundleItem, error) { previousUuid := "" previousArweaveID := "" if previous != nil { @@ -224,7 +232,7 @@ func arweave_bundle_single(pc *model.ProofChain, previous *model.ProofChain) (*a return &item, nil } -func revalidate_single(ctx context.Context, message *types.QueueMessage) error { +func revalidateSingle(ctx context.Context, message *types.QueueMessage) error { proof := model.Proof{} tx := model.DB.Preload("ProofChain").Preload("ProofChain.Previous").Where("id = ?", message.ProofID).First(&proof) if tx.Error != nil { @@ -233,7 +241,7 @@ func revalidate_single(ctx context.Context, message *types.QueueMessage) error { return proof.Revalidate() } -func init_db(cfg aws.Config) { +func initDB(cfg aws.Config) { model.Init(false) // TODO: should read auto migrate from ENV } @@ -241,7 +249,7 @@ func init_db(cfg aws.Config) { // sqs.Init(cfg) // } -func init_validators() { +func initValidators() { twitter.Init() ethereum.Init() keybase.Init() @@ -264,19 +272,19 @@ func init() { logrus.Fatalf("Unable to load AWS config: %s", err) } common.CurrentRuntime = common.Runtimes.Lambda - init_config_from_aws_secret() + initConfigFromAWSSecret() logrus.SetLevel(logrus.InfoLevel) - init_db(cfg) + initDB(cfg) // init_sqs(cfg) - init_validators() + initValidators() } -func init_config_from_aws_secret() { +func initConfigFromAWSSecret() { if initialized { return } - secret_name := getE("SECRET_NAME", "") + secretName := getE("SECRET_NAME", "") region := getE("SECRET_REGION", "") // Create a Secrets Manager client @@ -290,7 +298,7 @@ func init_config_from_aws_secret() { client := secretsmanager.NewFromConfig(cfg) input := secretsmanager.GetSecretValueInput{ - SecretId: aws.String(secret_name), + SecretId: aws.String(secretName), VersionStage: aws.String("AWSCURRENT"), } result, err := client.GetSecretValue(context.Background(), &input) @@ -319,13 +327,13 @@ func init_config_from_aws_secret() { initialized = true } -func getE(env_key, default_value string) string { - result := os.Getenv(env_key) +func getE(envKey, defaultValue string) string { + result := os.Getenv(envKey) if len(result) == 0 { - if len(default_value) > 0 { - return default_value + if len(defaultValue) > 0 { + return defaultValue } else { - logrus.Fatalf("ENV %s must be given! Abort.", env_key) + logrus.Fatalf("ENV %s must be given! Abort.", envKey) return "" } @@ -333,3 +341,19 @@ func getE(env_key, default_value string) string { return result } } + +func twitterRetrieveOAuthToken() (err error) { + type TokenList struct { + Tokens []twitter.Tokens `json:"tokens"` + } + // TODO: Retrieve existed token from a storage space (i.e., KV / S3) + + tokens, err := twitter.GenerateOauthToken() + if err != nil { + return err + } + fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens) + + // TODO: save new token to a storage space (i.e., KV / S3) + return nil +} diff --git a/types/mq.go b/types/mq.go index 0c6b8b3..bbd5bad 100644 --- a/types/mq.go +++ b/types/mq.go @@ -3,11 +3,13 @@ package types type QueueAction string var QueueActions = struct { - Revalidate QueueAction - ArweaveUpload QueueAction + Revalidate QueueAction + ArweaveUpload QueueAction + TwitterOAuthTokenAcquire QueueAction }{ - Revalidate: "revalidate", - ArweaveUpload: "arweave_upload", + Revalidate: "revalidate", + ArweaveUpload: "arweave_upload", + TwitterOAuthTokenAcquire: "twitter_oauth_token_acquire", } // QueueMessage indicates structure of messages in Amazon SQS. diff --git a/validator/twitter/api.go b/validator/twitter/api.go index ee88261..4b2577f 100644 --- a/validator/twitter/api.go +++ b/validator/twitter/api.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/nextdotid/proof_server/util" "github.com/samber/lo" "golang.org/x/xerrors" ) @@ -20,16 +21,18 @@ type APIResponse struct { Text string `json:"text"` } -const ( - BASIC_AUTH_USERNAME = "3rJOl1ODzm9yZy63FACdg" - BASIC_AUTH_PASSWORD = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" -) +type Tokens struct { + AccessToken string `json:"access_token"` + GuestToken string `json:"guest_token"` + FlowToken string `json:"flow_token"` + OAuthKey string `json:"oauth_key"` + OAuthSecret string `json:"oauth_secret"` + CreatedAt string `json:"created_at"` +} -var ( - // TODO: should save accessToken to somewhere else (shared by all Lambda instances) - accessToken string - guestToken string - flowToken string +const ( + TWITTER_CONSUMER_KEY = "3rJOl1ODzm9yZy63FACdg" + TWITTER_CONSUMER_SECRET = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" ) func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { @@ -38,7 +41,7 @@ func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) return nil, nil } -func setHeaders(req *http.Request, setAccessToken, setGuestToken bool) { +func setHeaders(req *http.Request, tokens *Tokens, setAccessToken, setGuestToken bool) { req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") @@ -49,20 +52,23 @@ func setHeaders(req *http.Request, setAccessToken, setGuestToken bool) { req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") req.Header.Set("X-Twitter-Active-User", "yes") if setGuestToken { - req.Header.Set("X-Guest-Token", guestToken) + req.Header.Set("X-Guest-Token", tokens.GuestToken) } if setAccessToken { - req.Header.Set("Authorization", "Bearer "+accessToken) + req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) } } -func GetOauthToken() (err error) { - if flowToken == "" { - if err := getFlowToken(); err != nil { - return err - } +// GenerateOauthToken generates a new Twitter OAuth guest token +// which can be used in calling Official APIs. +func GenerateOauthToken() (tokens *Tokens, err error) { + tokens = new(Tokens) + tokens.CreatedAt = util.TimeToTimestampString(time.Now()) + + if err := tokens.getFlowToken(); err != nil { + return nil, err } - l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", accessToken, guestToken, flowToken) + l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", tokens.AccessToken, tokens.GuestToken, tokens.FlowToken) requestBody := fmt.Sprintf(`{ "flow_token": "%s", @@ -128,22 +134,22 @@ func GetOauthToken() (err error) { "location_permission_prompt": 2, "notifications_permission_prompt": 4 } - }`, flowToken) + }`, tokens.FlowToken) req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) if err != nil { - return err + return nil, err } - setHeaders(req, true, true) + setHeaders(req, tokens, true, true) resp, err := new(http.Client).Do(req) if err != nil { - return err + return nil, err } body, err := io.ReadAll(resp.Body) if err != nil { - return err + return nil, err } l.Infof("Response: \n%s\n", body) type ResponseSubtask struct { @@ -163,36 +169,37 @@ func GetOauthToken() (err error) { // Should be "success" Status string `json:"status"` // A new flow token, usually ends with ":3" - FlowToken string `json:"flow_token"` + FlowToken string `json:"flow_token"` Subtasks []ResponseSubtask `json:"subtasks"` } response := new(Response) err = json.Unmarshal(body, response) if err != nil { - return err + return nil, err } if response.Errors != nil { - return xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) + return nil, xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) } if response.Status != "success" { - return xerrors.Errorf("wrong API status: %s", response.Status) + return nil, xerrors.Errorf("wrong API status: %s", response.Status) } st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { return (subtask.OpenAccount != nil) }) if !found { - return xerrors.Errorf("oauth token not found in response") + return nil, xerrors.Errorf("oauth token not found in response") } - flowToken = response.FlowToken + // Update new FlowToken + tokens.FlowToken = response.FlowToken l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) - return nil + return tokens, nil } -func getFlowToken() (err error) { - if guestToken == "" { - if err := getGuestToken(); err != nil { +func (tokens *Tokens) getFlowToken() (err error) { + if tokens.GuestToken == "" { + if err := tokens.getGuestToken(); err != nil { return err } } @@ -269,7 +276,7 @@ func getFlowToken() (err error) { if err != nil { return err } - setHeaders(req, true, true) + setHeaders(req, tokens, true, true) resp, err := new(http.Client).Do(req) if err != nil { @@ -293,16 +300,16 @@ func getFlowToken() (err error) { return xerrors.Errorf("empty FlowToken") } - flowToken = response.FlowToken + tokens.FlowToken = response.FlowToken return nil } -func getGuestToken() (err error) { - if guestToken != "" { +func (tokens *Tokens) getGuestToken() (err error) { + if tokens.GuestToken != "" { return nil } - if accessToken == "" { - if err = getAccessToken(); err != nil { + if tokens.AccessToken == "" { + if err = tokens.getAccessToken(); err != nil { return err } } @@ -310,7 +317,7 @@ func getGuestToken() (err error) { if err != nil { return err } - setHeaders(req, true, false) + setHeaders(req, tokens, true, false) type Response struct { GuestToken string `json:"guest_token"` } @@ -333,13 +340,13 @@ func getGuestToken() (err error) { if response.GuestToken == "" { return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) } - guestToken = response.GuestToken + tokens.GuestToken = response.GuestToken return nil } -func getAccessToken() (err error) { - if accessToken != "" { +func (tokens *Tokens) getAccessToken() (err error) { + if tokens.AccessToken != "" { return nil } @@ -351,8 +358,8 @@ func getAccessToken() (err error) { if err != nil { return err } - setHeaders(req, false, false) - req.SetBasicAuth(BASIC_AUTH_USERNAME, BASIC_AUTH_PASSWORD) + setHeaders(req, tokens, false, false) + req.SetBasicAuth(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET) resp, err := new(http.Client).Do(req) if err != nil { return err @@ -373,6 +380,13 @@ func getAccessToken() (err error) { return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) } - accessToken = response.AccessToken + tokens.AccessToken = response.AccessToken return nil } + +func (tokens *Tokens) IsExpired() bool { + const EXPIRED_AT = "24h" + expiredAt, _ := time.ParseDuration(EXPIRED_AT) + createdAt, _ := util.TimestampStringToTime(tokens.CreatedAt) + return createdAt.Add(expiredAt).Before(time.Now()) +} diff --git a/validator/twitter/api_test.go b/validator/twitter/api_test.go index 92b6108..d7748ef 100644 --- a/validator/twitter/api_test.go +++ b/validator/twitter/api_test.go @@ -8,33 +8,38 @@ import ( func Test_getAccessToken(t *testing.T) { t.Run("success", func(t *testing.T) { - require.NoError(t, getAccessToken()) - require.NotEmpty(t, accessToken) - t.Logf("Access token: %s", accessToken) + tokens := new(Tokens) + require.NoError(t, tokens.getAccessToken()) + require.NotEmpty(t, tokens.AccessToken) + t.Logf("Access token: %s", tokens.AccessToken) }) } func Test_getGuestToken(t *testing.T) { t.Run("success", func(t *testing.T) { - require.NoError(t, getGuestToken()) - require.NotEmpty(t, guestToken) - t.Logf("Guest token: %s", guestToken) + tokens := new(Tokens) + require.NoError(t, tokens.getGuestToken()) + require.NotEmpty(t, tokens.GuestToken) + t.Logf("Guest token: %s", tokens.GuestToken) }) } func Test_FlowToken(t *testing.T) { t.Run("success", func(t *testing.T) { - require.NoError(t, getFlowToken()) - require.NotEmpty(t, flowToken) - t.Logf("Flow token: %s", flowToken) + tokens := new(Tokens) + require.NoError(t, tokens.getFlowToken()) + require.NotEmpty(t, tokens.FlowToken) + t.Logf("Flow token: %s", tokens.FlowToken) }) } func Test_OauthToken(t *testing.T) { t.Run("success", func(t *testing.T) { - require.NoError(t, GetOauthToken()) - t.Logf("Access token: %s", accessToken) - t.Logf("Guest token: %s", guestToken) - t.Logf("Flow token: %s", flowToken) + tokens, err := GenerateOauthToken() + require.NoError(t, err) + require.False(t, tokens.IsExpired()) + t.Logf("Access token: %s", tokens.AccessToken) + t.Logf("Guest token: %s", tokens.GuestToken) + t.Logf("Flow token: %s", tokens.FlowToken) }) } From 3426ffb40ac9c29ec17e1d7f00cc9a1615e34c61 Mon Sep 17 00:00:00 2001 From: Nyk Ma Date: Sun, 17 Sep 2023 17:02:12 +0800 Subject: [PATCH 3/5] [#] lambda_worker: prefetching twitter OAuth key job --- cmd/lambda/main.go | 26 +- cmd/lambda_worker/main.go | 69 +-- go.mod | 16 +- go.sum | 27 +- util/s3/main.go | 47 ++ util/util.go | 18 + validator/twitter/api.go | 381 +--------------- validator/twitter/token.go | 418 ++++++++++++++++++ .../twitter/{api_test.go => token_test.go} | 6 +- 9 files changed, 580 insertions(+), 428 deletions(-) create mode 100644 util/s3/main.go create mode 100644 validator/twitter/token.go rename validator/twitter/{api_test.go => token_test.go} (93%) diff --git a/cmd/lambda/main.go b/cmd/lambda/main.go index 223d66d..504e300 100644 --- a/cmd/lambda/main.go +++ b/cmd/lambda/main.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "os" "github.com/akrylysov/algnhsa" "github.com/aws/aws-sdk-go-v2/aws" @@ -13,7 +12,9 @@ import ( myconfig "github.com/nextdotid/proof_server/config" "github.com/nextdotid/proof_server/controller" "github.com/nextdotid/proof_server/model" + "github.com/nextdotid/proof_server/util" "github.com/nextdotid/proof_server/util/sqs" + utilS3 "github.com/nextdotid/proof_server/util/s3" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" "github.com/nextdotid/proof_server/validator/discord" @@ -33,7 +34,8 @@ var ( ) func init_db(cfg aws.Config) { - model.Init(false) // TODO: should read auto migrate flag from ENV + shouldMigrate := util.GetE("DB_MIGRATE", "false") + model.Init(shouldMigrate == "true") } func init_sqs(cfg aws.Config) { @@ -70,6 +72,7 @@ func init() { init_db(cfg) init_sqs(cfg) init_validators() + utilS3.Init(cfg) controller.Init() } @@ -81,8 +84,8 @@ func init_config_from_aws_secret() { if initialized { return } - secret_name := getE("SECRET_NAME", "") - region := getE("SECRET_REGION", "") + secret_name := util.GetE("SECRET_NAME", "") + region := util.GetE("SECRET_REGION", "") // Create a Secrets Manager client cfg, err := config.LoadDefaultConfig( @@ -116,18 +119,3 @@ func init_config_from_aws_secret() { } initialized = true } - -func getE(env_key, default_value string) string { - result := os.Getenv(env_key) - if len(result) == 0 { - if len(default_value) > 0 { - return default_value - } else { - logrus.Fatalf("ENV %s must be given! Abort.", env_key) - return "" - } - - } else { - return result - } -} diff --git a/cmd/lambda_worker/main.go b/cmd/lambda_worker/main.go index 36027f3..b3aa680 100644 --- a/cmd/lambda_worker/main.go +++ b/cmd/lambda_worker/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "os" "strconv" "github.com/aws/aws-lambda-go/events" @@ -20,6 +19,8 @@ import ( myconfig "github.com/nextdotid/proof_server/config" "github.com/nextdotid/proof_server/model" "github.com/nextdotid/proof_server/types" + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" "github.com/nextdotid/proof_server/validator/discord" @@ -38,6 +39,7 @@ import ( ) var ( + awsConfig aws.Config initialized = false wallet *goar.Wallet ) @@ -81,7 +83,7 @@ func handler(ctx context.Context, sqs_event events.SQSEvent) (events.SQSEventRes } case types.QueueActions.TwitterOAuthTokenAcquire: { - err := twitterRetrieveOAuthToken() + err := twitterRefreshOAuthToken() if err != nil { // Ignore errors for now fmt.Printf("Error when retrieving Twitter OAuth key: %s", err.Error()) @@ -134,7 +136,7 @@ func arweave_upload_many(personas []string) error { for _, pc := range chains { if pc.ArweaveID != "" { - continue + } previous, ok := lo.Find(chains, func(item *model.ProofChain) bool { @@ -241,8 +243,9 @@ func revalidateSingle(ctx context.Context, message *types.QueueMessage) error { return proof.Revalidate() } -func initDB(cfg aws.Config) { - model.Init(false) // TODO: should read auto migrate from ENV +func initDB() { + shouldMigrate := util.GetE("DB_MIGRATE", "false") + model.Init(shouldMigrate == "true") } // func init_sqs(cfg aws.Config) { @@ -264,7 +267,8 @@ func initValidators() { } func init() { - cfg, err := config.LoadDefaultConfig( + var err error + awsConfig, err = config.LoadDefaultConfig( context.Background(), config.WithRegion("ap-east-1"), ) @@ -275,7 +279,7 @@ func init() { initConfigFromAWSSecret() logrus.SetLevel(logrus.InfoLevel) - initDB(cfg) + initDB() // init_sqs(cfg) initValidators() } @@ -284,8 +288,8 @@ func initConfigFromAWSSecret() { if initialized { return } - secretName := getE("SECRET_NAME", "") - region := getE("SECRET_REGION", "") + secretName := util.GetE("SECRET_NAME", "") + region := util.GetE("SECRET_REGION", "") // Create a Secrets Manager client cfg, err := config.LoadDefaultConfig( @@ -327,33 +331,34 @@ func initConfigFromAWSSecret() { initialized = true } -func getE(envKey, defaultValue string) string { - result := os.Getenv(envKey) - if len(result) == 0 { - if len(defaultValue) > 0 { - return defaultValue - } else { - logrus.Fatalf("ENV %s must be given! Abort.", envKey) - return "" - } - - } else { - return result +func twitterRefreshOAuthToken() (err error) { + const VALID_TOKEN_AMOUNT = 5 + if err = utilS3.Init(awsConfig); err != nil { + logrus.Fatalf("Error during initializing S3 client: %s", err.Error()) } -} - -func twitterRetrieveOAuthToken() (err error) { - type TokenList struct { - Tokens []twitter.Tokens `json:"tokens"` + ctx := context.Background() + tokens, err := twitter.GetTokenListFromS3(ctx) + if err != nil { + logrus.Fatalf("Error when loading Twitter token list from S3: %s", err.Error()) } - // TODO: Retrieve existed token from a storage space (i.e., KV / S3) - tokens, err := twitter.GenerateOauthToken() - if err != nil { - return err + validTokens := lo.Filter(tokens.Tokens, func(token twitter.Token, _index int) bool { + return !token.IsExpired() + }) + if len(validTokens) < VALID_TOKEN_AMOUNT { + // Generate a new one + newToken, err := twitter.GenerateOauthToken() + if err != nil { + return err + } + fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens) + validTokens = append(validTokens, *newToken) + newTokenList := twitter.TokenList{ + Tokens: validTokens, + } + newTokenListJSON, _ := newTokenList.ToJSON() + utilS3.PutToS3(ctx, twitter.TWITTER_TOKEN_LIST_FILENAME, newTokenListJSON) } - fmt.Printf("TWITTER OAUTH KEY REGISTERED: %+v", *tokens) - // TODO: save new token to a storage space (i.e., KV / S3) return nil } diff --git a/go.mod b/go.mod index 10e6e40..23ca0f8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/aws/aws-lambda-go v1.31.1 - github.com/aws/aws-sdk-go-v2 v1.16.5 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.15.4 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 github.com/everFinance/goar v1.4.2 @@ -26,15 +26,20 @@ require ( contrib.go.opencensus.io/exporter/stackdriver v0.13.4 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 // indirect - github.com/aws/smithy-go v1.11.3 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect github.com/blendle/zapdriver v1.3.1 // indirect github.com/btcsuite/btcd v0.22.0-beta // indirect @@ -148,6 +153,7 @@ require ( require ( github.com/akrylysov/algnhsa v0.12.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 github.com/bwmarrin/discordgo v0.25.0 github.com/ethereum/go-ethereum v1.10.25 diff --git a/go.sum b/go.sum index d9fe55d..c58018d 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,11 @@ github.com/aws/aws-lambda-go v1.31.1/go.mod h1:IF5Q7wj4VyZyUFnZ54IQqeWtctHQ9tz+K github.com/aws/aws-sdk-go v1.22.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= github.com/aws/aws-sdk-go-v2/config v1.15.4 h1:P4mesY1hYUxru4f9SU0XxNKXmzfxsD0FtMIPRBjkH7Q= github.com/aws/aws-sdk-go-v2/config v1.15.4/go.mod h1:ZijHHh0xd/A+ZY53az0qzC5tT46kt4JVCePf2NX9Lk4= github.com/aws/aws-sdk-go-v2/credentials v1.12.0 h1:4R/NqlcRFSkR0wxOhgHi+agGpbEr5qMCjn7VqUIJY+E= @@ -84,15 +87,28 @@ github.com/aws/aws-sdk-go-v2/credentials v1.12.0/go.mod h1:9YWk7VW+eyKsoIL6/Cljk github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4 h1:FP8gquGeGHHdfY6G5llaMQDF+HAf20VKc8opRwmjf04= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.4/go.mod h1:u/s5/Z+ohUQOPXl00m2yJVyioWDECsbpXTQlaqSlufc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11 h1:6cZRymlLEIlDTEB0+5+An6Zj1CKt6rSE69tOmFeu1nk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.11/go.mod h1:0MR+sS1b/yxsfAPvAESrw8NfwUoxMinDyw6EYR9BS2U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 h1:b16QW0XWl0jWjLABFc1A+uh145Oqv+xDcObNk0iQgUk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 h1:m+mxqLIrGq7GJo5qw4rHn8BbUqHrvxvwFx54N1Pglvw= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6/go.mod h1:Z+i6uqZgCOBXhNoEGoRm/ZaLsaJA9rGUAmkVKM/3+g4= github.com/aws/aws-sdk-go-v2/service/sqs v1.18.6 h1:HlEYt9p1TAQYxeB8jz3y4dmXmZevX+cJnh8OU6x0aqo= @@ -102,8 +118,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.4/go.mod h1:cPDwJwsP4Kff9mldCXAmd github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 h1:+xtV90n3abQmgzk1pS++FdxZTrPEDgQng6e4/56WR2A= github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= diff --git a/util/s3/main.go b/util/s3/main.go new file mode 100644 index 0000000..fb9d88e --- /dev/null +++ b/util/s3/main.go @@ -0,0 +1,47 @@ +package s3 + +import ( + "bytes" + "context" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/nextdotid/proof_server/util" +) + +var ( + s3Client *s3.Client + bucketName string +) + +func Init(awsConfig aws.Config) error { + if s3Client == nil { + s3Client = s3.NewFromConfig(awsConfig) + } + bucketName = util.GetE("S3_BUCKET", "") + return nil +} + +// ReadFromS3 should be called before Init() +func ReadFromS3(ctx context.Context, key string) (content []byte, err error) { + result, err := s3Client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err != nil { + return []byte{}, err + } + defer result.Body.Close() + return io.ReadAll(result.Body) +} + +// PutToS3 should be called before Init() +func PutToS3(ctx context.Context, key string, content []byte) (err error) { + _, err = s3Client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: bytes.NewReader(content), + }) + return err +} diff --git a/util/util.go b/util/util.go index 6a06e8e..cc5f5bc 100644 --- a/util/util.go +++ b/util/util.go @@ -2,10 +2,12 @@ package util import ( "encoding/base64" + "os" "strconv" "time" "github.com/nextdotid/proof_server/util/base1024" + "github.com/sirupsen/logrus" "golang.org/x/xerrors" ) @@ -30,3 +32,19 @@ func DecodeString(s string) ([]byte, error) { } return base1024.DecodeString(s) } + +// GetE gets current system's environment variable as `string`. +// If `defaultValue` is empty, this environment key must be exist, or it will panic. +func GetE(envKey, defaultValue string) string { + result := os.Getenv(envKey) + if len(result) == 0 { + if len(defaultValue) > 0 { + return defaultValue + } else { + logrus.Fatalf("ENV %s must be given! Abort.", envKey) + return "" + } + + } + return result +} diff --git a/validator/twitter/api.go b/validator/twitter/api.go index 4b2577f..12348a1 100644 --- a/validator/twitter/api.go +++ b/validator/twitter/api.go @@ -1,14 +1,9 @@ package twitter import ( - "encoding/json" - "fmt" - "io" - "net/http" - "strings" + "context" "time" - "github.com/nextdotid/proof_server/util" "github.com/samber/lo" "golang.org/x/xerrors" ) @@ -21,372 +16,30 @@ type APIResponse struct { Text string `json:"text"` } -type Tokens struct { - AccessToken string `json:"access_token"` - GuestToken string `json:"guest_token"` - FlowToken string `json:"flow_token"` - OAuthKey string `json:"oauth_key"` - OAuthSecret string `json:"oauth_secret"` - CreatedAt string `json:"created_at"` -} - -const ( - TWITTER_CONSUMER_KEY = "3rJOl1ODzm9yZy63FACdg" - TWITTER_CONSUMER_SECRET = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" +var ( + CurrentTokenList *TokenList ) func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { const RETRY_AFTER = time.Second - - return nil, nil -} - -func setHeaders(req *http.Request, tokens *Tokens, setAccessToken, setGuestToken bool) { - req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - req.Header.Set("X-Twitter-API-Version", "5") - req.Header.Set("X-Twitter-Client", "TwitterAndroid") - req.Header.Set("X-Twitter-Client-Version", "9.95.0-release.0") - req.Header.Set("OS-Version", "28") - req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") - req.Header.Set("X-Twitter-Active-User", "yes") - if setGuestToken { - req.Header.Set("X-Guest-Token", tokens.GuestToken) - } - if setAccessToken { - req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) - } -} - -// GenerateOauthToken generates a new Twitter OAuth guest token -// which can be used in calling Official APIs. -func GenerateOauthToken() (tokens *Tokens, err error) { - tokens = new(Tokens) - tokens.CreatedAt = util.TimeToTimestampString(time.Now()) - - if err := tokens.getFlowToken(); err != nil { - return nil, err - } - l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", tokens.AccessToken, tokens.GuestToken, tokens.FlowToken) - - requestBody := fmt.Sprintf(`{ - "flow_token": "%s", - "subtask_inputs": [ - { - "open_link": { - "link": "next_link" - }, - "subtask_id": "NextTaskOpenLink" - } - ], - "subtask_versions": { - "generic_urt": 3, - "standard": 1, - "open_home_timeline": 1, - "app_locale_update": 1, - "enter_date": 1, - "email_verification": 3, - "enter_password": 5, - "enter_text": 5, - "one_tap": 2, - "cta": 7, - "single_sign_on": 1, - "fetch_persisted_data": 1, - "enter_username": 3, - "web_modal": 2, - "fetch_temporary_password": 1, - "menu_dialog": 1, - "sign_up_review": 5, - "interest_picker": 4, - "user_recommendations_urt": 3, - "in_app_notification": 1, - "sign_up": 2, - "typeahead_search": 1, - "user_recommendations_list": 4, - "cta_inline": 1, - "contacts_live_sync_permission_prompt": 3, - "choice_selection": 5, - "js_instrumentation": 1, - "alert_dialog_suppress_client_events": 1, - "privacy_options": 1, - "topics_selector": 1, - "wait_spinner": 3, - "tweet_selection_urt": 1, - "end_flow": 1, - "settings_list": 7, - "open_external_link": 1, - "phone_verification": 5, - "security_key": 3, - "select_banner": 2, - "upload_media": 1, - "web": 2, - "alert_dialog": 1, - "open_account": 2, - "action_list": 2, - "enter_phone": 2, - "open_link": 1, - "show_code": 1, - "update_users": 1, - "check_logged_in_account": 1, - "enter_email": 2, - "select_avatar": 4, - "location_permission_prompt": 2, - "notifications_permission_prompt": 4 - } - }`, tokens.FlowToken) - - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) - if err != nil { - return nil, err - } - setHeaders(req, tokens, true, true) - - resp, err := new(http.Client).Do(req) - if err != nil { - return nil, err - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - l.Infof("Response: \n%s\n", body) - type ResponseSubtask struct { - // Should exist - OpenAccount *struct { - OauthToken string `json:"oauth_token"` - OauthTokenSecret string `json:"oauth_token_secret"` - } `json:"open_account"` - } - - type Response struct { - // Should be empty - Errors *[]struct { - Code int `json:"code"` - Message string `json:"message"` - } `json:"errors"` - // Should be "success" - Status string `json:"status"` - // A new flow token, usually ends with ":3" - FlowToken string `json:"flow_token"` - Subtasks []ResponseSubtask `json:"subtasks"` - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return nil, err - } - if response.Errors != nil { - return nil, xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) - } - if response.Status != "success" { - return nil, xerrors.Errorf("wrong API status: %s", response.Status) - } - - st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { - return (subtask.OpenAccount != nil) - }) - if !found { - return nil, xerrors.Errorf("oauth token not found in response") - } - // Update new FlowToken - tokens.FlowToken = response.FlowToken - l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) - - return tokens, nil -} - -func (tokens *Tokens) getFlowToken() (err error) { - if tokens.GuestToken == "" { - if err := tokens.getGuestToken(); err != nil { - return err + ctx := context.Background() + if CurrentTokenList == nil { + CurrentTokenList, err = GetTokenListFromS3(ctx) + if err != nil { + return nil, xerrors.Errorf("fetchPostWithAPI: %w", err) } - } - - requestBody := `{ - "flow_token": null, - "input_flow_data": { - "country_code": null, - "flow_context": { - "start_location": { - "location": "splash_screen" - } - }, - "requested_variant": null, - "target_user_id": 0 - }, - "subtask_versions": { - "generic_urt": 3, - "standard": 1, - "open_home_timeline": 1, - "app_locale_update": 1, - "enter_date": 1, - "email_verification": 3, - "enter_password": 5, - "enter_text": 5, - "one_tap": 2, - "cta": 7, - "single_sign_on": 1, - "fetch_persisted_data": 1, - "enter_username": 3, - "web_modal": 2, - "fetch_temporary_password": 1, - "menu_dialog": 1, - "sign_up_review": 5, - "interest_picker": 4, - "user_recommendations_urt": 3, - "in_app_notification": 1, - "sign_up": 2, - "typeahead_search": 1, - "user_recommendations_list": 4, - "cta_inline": 1, - "contacts_live_sync_permission_prompt": 3, - "choice_selection": 5, - "js_instrumentation": 1, - "alert_dialog_suppress_client_events": 1, - "privacy_options": 1, - "topics_selector": 1, - "wait_spinner": 3, - "tweet_selection_urt": 1, - "end_flow": 1, - "settings_list": 7, - "open_external_link": 1, - "phone_verification": 5, - "security_key": 3, - "select_banner": 2, - "upload_media": 1, - "web": 2, - "alert_dialog": 1, - "open_account": 2, - "action_list": 2, - "enter_phone": 2, - "open_link": 1, - "show_code": 1, - "update_users": 1, - "check_logged_in_account": 1, - "enter_email": 2, - "select_avatar": 4, - "location_permission_prompt": 2, - "notifications_permission_prompt": 4 - } - }` - - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json?flow_name=welcome&api_version=1&known_device_token=&sim_country_code=us", strings.NewReader(requestBody)) - if err != nil { - return err - } - setHeaders(req, tokens, true, true) - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - type Response struct { - FlowToken string `json:"flow_token"` - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return err - } - - if response.FlowToken == "" { - return xerrors.Errorf("empty FlowToken") - } - - tokens.FlowToken = response.FlowToken - return nil -} - -func (tokens *Tokens) getGuestToken() (err error) { - if tokens.GuestToken != "" { - return nil - } - if tokens.AccessToken == "" { - if err = tokens.getAccessToken(); err != nil { - return err + if CurrentTokenList == nil { + return nil, xerrors.Errorf("twitter token list does not exist") } } - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) - if err != nil { - return err - } - setHeaders(req, tokens, true, false) - type Response struct { - GuestToken string `json:"guest_token"` - } - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - - // Fetching bearerToken - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return err - } - if response.GuestToken == "" { - return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) + token := lo.Sample(CurrentTokenList.Tokens) + if lo.IsEmpty(token.OAuthSecret) || lo.IsEmpty(token.OAuthKey) { + return nil, xerrors.Errorf("twitter token seems to be empty") } - tokens.GuestToken = response.GuestToken - return nil -} - -func (tokens *Tokens) getAccessToken() (err error) { - if tokens.AccessToken != "" { - return nil - } - - type Response struct { - TokenType string `json:"token_type"` - AccessToken string `json:"access_token"` - } - req, err := http.NewRequest("POST", "https://api.twitter.com/oauth2/token?grant_type=client_credentials", nil) - if err != nil { - return err - } - setHeaders(req, tokens, false, false) - req.SetBasicAuth(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET) - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } + // TODO: Use token to query specific tweet with twitter API + // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets + // https://api.twitter.com/1.1/statuses/show.json - // Fetching bearerToken - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - response := new(Response) - err = json.Unmarshal(body, response) - if err != nil { - return err - } - - if response.TokenType != "bearer" || len(response.AccessToken) == 0 { - return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) - } - - tokens.AccessToken = response.AccessToken - return nil -} - -func (tokens *Tokens) IsExpired() bool { - const EXPIRED_AT = "24h" - expiredAt, _ := time.ParseDuration(EXPIRED_AT) - createdAt, _ := util.TimestampStringToTime(tokens.CreatedAt) - return createdAt.Add(expiredAt).Before(time.Now()) + return nil, nil } diff --git a/validator/twitter/token.go b/validator/twitter/token.go new file mode 100644 index 0000000..a798dd8 --- /dev/null +++ b/validator/twitter/token.go @@ -0,0 +1,418 @@ +package twitter + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/nextdotid/proof_server/util" + utilS3 "github.com/nextdotid/proof_server/util/s3" + "github.com/samber/lo" + "golang.org/x/xerrors" +) + +const ( + TWITTER_TOKEN_LIST_FILENAME string = "config/proof_service/twitter_oauth_tokens.json" +) + +type Token struct { + AccessToken string `json:"access_token"` + GuestToken string `json:"guest_token"` + FlowToken string `json:"flow_token"` + OAuthKey string `json:"oauth_key"` + OAuthSecret string `json:"oauth_secret"` + CreatedAt string `json:"created_at"` +} + +type TokenList struct { + Tokens []Token `json:"tokens"` +} + +const ( + TWITTER_CONSUMER_KEY = "3rJOl1ODzm9yZy63FACdg" + TWITTER_CONSUMER_SECRET = "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8" +) + +func setHeaders(req *http.Request, tokens *Token, setAccessToken, setGuestToken bool) { + req.Header.Set("User-Agent", "TwitterAndroid/9.95.0-release.0 (29950000-r-0) ONEPLUS+A3010/9 (OnePlus;ONEPLUS+A3010;OnePlus;OnePlus3;0;;1;2016)") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Twitter-API-Version", "5") + req.Header.Set("X-Twitter-Client", "TwitterAndroid") + req.Header.Set("X-Twitter-Client-Version", "9.95.0-release.0") + req.Header.Set("OS-Version", "28") + req.Header.Set("System-User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; ONEPLUS A3010 Build/PKQ1.181203.001)") + req.Header.Set("X-Twitter-Active-User", "yes") + if setGuestToken { + req.Header.Set("X-Guest-Token", tokens.GuestToken) + } + if setAccessToken { + req.Header.Set("Authorization", "Bearer "+tokens.AccessToken) + } +} + +// GenerateOauthToken generates a new Twitter OAuth guest token +// which can be used in calling Official APIs. +func GenerateOauthToken() (token *Token, err error) { + token = new(Token) + token.CreatedAt = util.TimeToTimestampString(time.Now()) + + if err := token.getFlowToken(); err != nil { + return nil, err + } + l.Infof("Access token: %s\nGuest token: %s\nFlow token: %s\n", token.AccessToken, token.GuestToken, token.FlowToken) + + requestBody := fmt.Sprintf(`{ + "flow_token": "%s", + "subtask_inputs": [ + { + "open_link": { + "link": "next_link" + }, + "subtask_id": "NextTaskOpenLink" + } + ], + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }`, token.FlowToken) + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json", strings.NewReader(requestBody)) + if err != nil { + return nil, err + } + setHeaders(req, token, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return nil, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + l.Infof("Response: \n%s\n", body) + type ResponseSubtask struct { + // Should exist + OpenAccount *struct { + OauthToken string `json:"oauth_token"` + OauthTokenSecret string `json:"oauth_token_secret"` + } `json:"open_account"` + } + + type Response struct { + // Should be empty + Errors *[]struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"errors"` + // Should be "success" + Status string `json:"status"` + // A new flow token, usually ends with ":3" + FlowToken string `json:"flow_token"` + Subtasks []ResponseSubtask `json:"subtasks"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return nil, err + } + if response.Errors != nil { + return nil, xerrors.Errorf("error when getting oauth token: %+v", *response.Errors) + } + if response.Status != "success" { + return nil, xerrors.Errorf("wrong API status: %s", response.Status) + } + + st, found := lo.Find(response.Subtasks, func(subtask ResponseSubtask) bool { + return (subtask.OpenAccount != nil) + }) + if !found { + return nil, xerrors.Errorf("oauth token not found in response") + } + // Update new FlowToken + token.FlowToken = response.FlowToken + l.Infof("OAUTH TOKEN REGISTERED: %s --- %s", st.OpenAccount.OauthToken, st.OpenAccount.OauthTokenSecret) + + return token, nil +} + +func (token *Token) getFlowToken() (err error) { + if token.GuestToken == "" { + if err := token.getGuestToken(); err != nil { + return err + } + } + + requestBody := `{ + "flow_token": null, + "input_flow_data": { + "country_code": null, + "flow_context": { + "start_location": { + "location": "splash_screen" + } + }, + "requested_variant": null, + "target_user_id": 0 + }, + "subtask_versions": { + "generic_urt": 3, + "standard": 1, + "open_home_timeline": 1, + "app_locale_update": 1, + "enter_date": 1, + "email_verification": 3, + "enter_password": 5, + "enter_text": 5, + "one_tap": 2, + "cta": 7, + "single_sign_on": 1, + "fetch_persisted_data": 1, + "enter_username": 3, + "web_modal": 2, + "fetch_temporary_password": 1, + "menu_dialog": 1, + "sign_up_review": 5, + "interest_picker": 4, + "user_recommendations_urt": 3, + "in_app_notification": 1, + "sign_up": 2, + "typeahead_search": 1, + "user_recommendations_list": 4, + "cta_inline": 1, + "contacts_live_sync_permission_prompt": 3, + "choice_selection": 5, + "js_instrumentation": 1, + "alert_dialog_suppress_client_events": 1, + "privacy_options": 1, + "topics_selector": 1, + "wait_spinner": 3, + "tweet_selection_urt": 1, + "end_flow": 1, + "settings_list": 7, + "open_external_link": 1, + "phone_verification": 5, + "security_key": 3, + "select_banner": 2, + "upload_media": 1, + "web": 2, + "alert_dialog": 1, + "open_account": 2, + "action_list": 2, + "enter_phone": 2, + "open_link": 1, + "show_code": 1, + "update_users": 1, + "check_logged_in_account": 1, + "enter_email": 2, + "select_avatar": 4, + "location_permission_prompt": 2, + "notifications_permission_prompt": 4 + } + }` + + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/onboarding/task.json?flow_name=welcome&api_version=1&known_device_token=&sim_country_code=us", strings.NewReader(requestBody)) + if err != nil { + return err + } + setHeaders(req, token, true, true) + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + type Response struct { + FlowToken string `json:"flow_token"` + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.FlowToken == "" { + return xerrors.Errorf("empty FlowToken") + } + + token.FlowToken = response.FlowToken + return nil +} + +func (token *Token) getGuestToken() (err error) { + if token.GuestToken != "" { + return nil + } + if token.AccessToken == "" { + if err = token.getAccessToken(); err != nil { + return err + } + } + req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) + if err != nil { + return err + } + setHeaders(req, token, true, false) + type Response struct { + GuestToken string `json:"guest_token"` + } + + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + if response.GuestToken == "" { + return xerrors.Errorf("Wrong guest token: %s", response.GuestToken) + } + token.GuestToken = response.GuestToken + + return nil +} + +func (token *Token) getAccessToken() (err error) { + if token.AccessToken != "" { + return nil + } + + type Response struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + } + req, err := http.NewRequest("POST", "https://api.twitter.com/oauth2/token?grant_type=client_credentials", nil) + if err != nil { + return err + } + setHeaders(req, token, false, false) + req.SetBasicAuth(TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET) + resp, err := new(http.Client).Do(req) + if err != nil { + return err + } + + // Fetching bearerToken + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + response := new(Response) + err = json.Unmarshal(body, response) + if err != nil { + return err + } + + if response.TokenType != "bearer" || len(response.AccessToken) == 0 { + return xerrors.Errorf("Wrong bearer token: %s %s", response.TokenType, response.AccessToken) + } + + token.AccessToken = response.AccessToken + return nil +} + +func (token *Token) IsExpired() bool { + const EXPIRED_AT = "24h" + expiredAt, _ := time.ParseDuration(EXPIRED_AT) + createdAt, _ := util.TimestampStringToTime(token.CreatedAt) + return createdAt.Add(expiredAt).Before(time.Now()) +} + +func (tl *TokenList) ToJSON() ([]byte, error) { + return json.Marshal(*tl) +} + +func TokenListFromJSON(content []byte) (*TokenList, error) { + tl := new(TokenList) + err := json.Unmarshal(content, tl) + if err != nil { + return nil, err + } + return tl, nil +} + +// GetTokenListFromS3 reads JSON file in AWS S3 to get token lists. +func GetTokenListFromS3(ctx context.Context) (*TokenList, error) { + body, _ := utilS3.ReadFromS3(ctx, TWITTER_TOKEN_LIST_FILENAME) + // If file not found, touch a new one: + if len(body) == 0 { + tl := new(TokenList) + newTokenList, _ := tl.ToJSON() + if err := utilS3.PutToS3(ctx, TWITTER_TOKEN_LIST_FILENAME, newTokenList); err != nil { + return nil, err + } + return tl, nil + } + + // File found, read it + return TokenListFromJSON(body) +} diff --git a/validator/twitter/api_test.go b/validator/twitter/token_test.go similarity index 93% rename from validator/twitter/api_test.go rename to validator/twitter/token_test.go index d7748ef..d671f37 100644 --- a/validator/twitter/api_test.go +++ b/validator/twitter/token_test.go @@ -8,7 +8,7 @@ import ( func Test_getAccessToken(t *testing.T) { t.Run("success", func(t *testing.T) { - tokens := new(Tokens) + tokens := new(Token) require.NoError(t, tokens.getAccessToken()) require.NotEmpty(t, tokens.AccessToken) t.Logf("Access token: %s", tokens.AccessToken) @@ -17,7 +17,7 @@ func Test_getAccessToken(t *testing.T) { func Test_getGuestToken(t *testing.T) { t.Run("success", func(t *testing.T) { - tokens := new(Tokens) + tokens := new(Token) require.NoError(t, tokens.getGuestToken()) require.NotEmpty(t, tokens.GuestToken) t.Logf("Guest token: %s", tokens.GuestToken) @@ -26,7 +26,7 @@ func Test_getGuestToken(t *testing.T) { func Test_FlowToken(t *testing.T) { t.Run("success", func(t *testing.T) { - tokens := new(Tokens) + tokens := new(Token) require.NoError(t, tokens.getFlowToken()) require.NotEmpty(t, tokens.FlowToken) t.Logf("Flow token: %s", tokens.FlowToken) From 2e0f4f2c3faa0c28d4a6cb1d9c427e634fba7e17 Mon Sep 17 00:00:00 2001 From: Nyk Ma Date: Sun, 17 Sep 2023 21:13:29 +0800 Subject: [PATCH 4/5] [#] validator/twitter: use Twitter API v2 as tweet fetcher --- config/main.go | 6 +++ go.mod | 1 + go.sum | 2 + validator/twitter/api.go | 88 +++++++++++++++++++++++++++-------- validator/twitter/api_test.go | 17 +++++++ 5 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 validator/twitter/api_test.go diff --git a/config/main.go b/config/main.go index 25c74d8..f2d8a4e 100644 --- a/config/main.go +++ b/config/main.go @@ -33,12 +33,18 @@ type HeadlessConfig struct { } type PlatformConfig struct { + Twitter TwitterPlatformConfig `json:"twitter"` Telegram TelegramPlatformConfig `json:"telegram"` Ethereum EthereumPlatformConfig `json:"ethereum"` Discord DiscordPlatformConfig `json:"discord"` Slack SlackPlatformConfig `json:"slack"` } +type TwitterPlatformConfig struct { + // Twitter API v2 Bearer token + OauthToken string `json:"oauth_token"` +} + type ArweaveConfig struct { Jwk string `json:"jwk"` ClientUrl string `json:"client_url"` diff --git a/go.mod b/go.mod index 23ca0f8..21f69a0 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/everFinance/ttcrsa v1.1.3 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.5.3 // indirect + github.com/g8rswimmer/go-twitter/v2 v2.1.5 // indirect github.com/gagliardetto/binary v0.6.1 // indirect github.com/gagliardetto/treeout v0.1.4 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index c58018d..6fa4dce 100644 --- a/go.sum +++ b/go.sum @@ -223,6 +223,8 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.3 h1:vNFpj2z7YIbwh2bw7x35sqYpp2wfuq+pivKbWG09B8c= github.com/fsnotify/fsnotify v1.5.3/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/g8rswimmer/go-twitter/v2 v2.1.5 h1:Uj9Yuof2UducrP4Xva7irnUJfB9354/VyUXKmc2D5gg= +github.com/g8rswimmer/go-twitter/v2 v2.1.5/go.mod h1:/55xWb313KQs25X7oZrNSEwLQNkYHhPsDwFstc45vhc= github.com/gagliardetto/binary v0.6.1 h1:vGrbUym10xaaswadfnuSDr0xlP3NZS5XWbLqENJidrI= github.com/gagliardetto/binary v0.6.1/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0= github.com/gagliardetto/gofuzz v1.2.2/go.mod h1:bkH/3hYLZrMLbfYWA0pWzXmi5TTRZnu4pMGZBkqMKvY= diff --git a/validator/twitter/api.go b/validator/twitter/api.go index 12348a1..d53d110 100644 --- a/validator/twitter/api.go +++ b/validator/twitter/api.go @@ -2,9 +2,12 @@ package twitter import ( "context" - "time" + "fmt" + "net/http" + "strings" - "github.com/samber/lo" + twitter "github.com/g8rswimmer/go-twitter/v2" + "github.com/nextdotid/proof_server/config" "golang.org/x/xerrors" ) @@ -17,29 +20,76 @@ type APIResponse struct { } var ( + twitterClient *twitter.Client CurrentTokenList *TokenList ) -func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { - const RETRY_AFTER = time.Second - ctx := context.Background() - if CurrentTokenList == nil { - CurrentTokenList, err = GetTokenListFromS3(ctx) - if err != nil { - return nil, xerrors.Errorf("fetchPostWithAPI: %w", err) - } - if CurrentTokenList == nil { - return nil, xerrors.Errorf("twitter token list does not exist") +type authorize struct { + Token string +} + +func (a authorize) Add(req *http.Request) { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", a.Token)) +} + +// Fetch tweet using twitter OAuth2.0 API. +// FIXME: should be switched to guest OAuth token solution. +func fetchPostWithAPI(id string, maxRetries int) (*APIResponse, error) { + if twitterClient == nil { + twitterClient = &twitter.Client{ + Authorizer: authorize{ + Token: config.C.Platform.Twitter.OauthToken, + }, + Client: http.DefaultClient, + Host: "https://api.twitter.com", } } - token := lo.Sample(CurrentTokenList.Tokens) - if lo.IsEmpty(token.OAuthSecret) || lo.IsEmpty(token.OAuthKey) { - return nil, xerrors.Errorf("twitter token seems to be empty") + opts := twitter.TweetLookupOpts{ + Expansions: []twitter.Expansion{twitter.ExpansionEntitiesMentionsUserName, twitter.ExpansionAuthorID}, + TweetFields: []twitter.TweetField{twitter.TweetFieldText, twitter.TweetFieldCreatedAt, twitter.TweetFieldEntities}, + } + result, err := twitterClient.TweetLookup(context.Background(), []string{id}, opts) + if err != nil { + return nil, xerrors.Errorf("error when retriving tweet: %w", err) + } + tweet := result.Raw.Tweets[0] + if tweet == nil { + return nil, xerrors.Errorf("tweet not found: %s", id) } - // TODO: Use token to query specific tweet with twitter API - // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets - // https://api.twitter.com/1.1/statuses/show.json + response := APIResponse{ + Text: tweet.Text, + } + response.User.ID = tweet.AuthorID + if len(tweet.Entities.Mentions) > 0 { + // Expect to be the user himself + mention := tweet.Entities.Mentions[0] + response.User.ScreenName = strings.ToLower(mention.UserName) + } - return nil, nil + return &response, nil } + +// func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { +// const RETRY_AFTER = time.Second +// ctx := context.Background() +// if CurrentTokenList == nil { +// CurrentTokenList, err = GetTokenListFromS3(ctx) +// if err != nil { +// return nil, xerrors.Errorf("fetchPostWithAPI: %w", err) +// } +// if CurrentTokenList == nil { +// return nil, xerrors.Errorf("twitter token list does not exist") +// } +// } +// token := lo.Sample(CurrentTokenList.Tokens) +// if lo.IsEmpty(token.OAuthSecret) || lo.IsEmpty(token.OAuthKey) { +// return nil, xerrors.Errorf("twitter token seems to be empty") +// } + +// // TODO: Use token to query specific tweet with twitter API +// // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets +// // https://api.twitter.com/1.1/statuses/show.json + +// return nil, nil +// } diff --git a/validator/twitter/api_test.go b/validator/twitter/api_test.go new file mode 100644 index 0000000..f39f2cd --- /dev/null +++ b/validator/twitter/api_test.go @@ -0,0 +1,17 @@ +package twitter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fetchPostWithAPI(t *testing.T) { + t.Run("success", func(t *testing.T) { + tweet, err := fetchPostWithAPI("1652176440396517378", 10) + require.NoError(t, err) + require.Contains(t, tweet.Text, "Sig:") + require.Equal(t, tweet.User.ScreenName, "bgm38") + require.Equal(t, tweet.User.ID, "292254624") + }) +} From cfa540d8ffe36101ab7798ec6a199c773a490772 Mon Sep 17 00:00:00 2001 From: Nyk Ma Date: Sun, 17 Sep 2023 21:17:38 +0800 Subject: [PATCH 5/5] [!] misc: format --- cmd/lambda/main.go | 4 ++-- util/s3/main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lambda/main.go b/cmd/lambda/main.go index 504e300..8306904 100644 --- a/cmd/lambda/main.go +++ b/cmd/lambda/main.go @@ -13,8 +13,8 @@ import ( "github.com/nextdotid/proof_server/controller" "github.com/nextdotid/proof_server/model" "github.com/nextdotid/proof_server/util" - "github.com/nextdotid/proof_server/util/sqs" utilS3 "github.com/nextdotid/proof_server/util/s3" + "github.com/nextdotid/proof_server/util/sqs" "github.com/nextdotid/proof_server/validator/activitypub" "github.com/nextdotid/proof_server/validator/das" "github.com/nextdotid/proof_server/validator/discord" @@ -72,7 +72,7 @@ func init() { init_db(cfg) init_sqs(cfg) init_validators() - utilS3.Init(cfg) + // utilS3.Init(cfg) controller.Init() } diff --git a/util/s3/main.go b/util/s3/main.go index fb9d88e..a100f9a 100644 --- a/util/s3/main.go +++ b/util/s3/main.go @@ -11,7 +11,7 @@ import ( ) var ( - s3Client *s3.Client + s3Client *s3.Client bucketName string )