diff --git a/config/main.go b/config/main.go index 25c74d8..f2d8a4e 100644 --- a/config/main.go +++ b/config/main.go @@ -33,12 +33,18 @@ type HeadlessConfig struct { } type PlatformConfig struct { + Twitter TwitterPlatformConfig `json:"twitter"` Telegram TelegramPlatformConfig `json:"telegram"` Ethereum EthereumPlatformConfig `json:"ethereum"` Discord DiscordPlatformConfig `json:"discord"` Slack SlackPlatformConfig `json:"slack"` } +type TwitterPlatformConfig struct { + // Twitter API v2 Bearer token + OauthToken string `json:"oauth_token"` +} + type ArweaveConfig struct { Jwk string `json:"jwk"` ClientUrl string `json:"client_url"` diff --git a/go.mod b/go.mod index 10e6e40..c2b9c38 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.15.4 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.6 github.com/everFinance/goar v1.4.2 + github.com/g8rswimmer/go-twitter/v2 v2.1.5 github.com/gagliardetto/solana-go v1.4.0 github.com/gin-gonic/gin v1.7.7 github.com/go-faster/errors v0.6.1 diff --git a/go.sum b/go.sum index d9fe55d..bec17b3 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,8 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.3 h1:vNFpj2z7YIbwh2bw7x35sqYpp2wfuq+pivKbWG09B8c= github.com/fsnotify/fsnotify v1.5.3/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/g8rswimmer/go-twitter/v2 v2.1.5 h1:Uj9Yuof2UducrP4Xva7irnUJfB9354/VyUXKmc2D5gg= +github.com/g8rswimmer/go-twitter/v2 v2.1.5/go.mod h1:/55xWb313KQs25X7oZrNSEwLQNkYHhPsDwFstc45vhc= github.com/gagliardetto/binary v0.6.1 h1:vGrbUym10xaaswadfnuSDr0xlP3NZS5XWbLqENJidrI= github.com/gagliardetto/binary v0.6.1/go.mod h1:aOfYkc20U0deHaHn/LVZXiqlkDbFAX0FpTlDhsXa0S0= github.com/gagliardetto/gofuzz v1.2.2/go.mod h1:bkH/3hYLZrMLbfYWA0pWzXmi5TTRZnu4pMGZBkqMKvY= diff --git a/validator/twitter/api.go b/validator/twitter/api.go new file mode 100644 index 0000000..387fb3d --- /dev/null +++ b/validator/twitter/api.go @@ -0,0 +1,116 @@ +package twitter + +import ( + "context" + "fmt" + "net/http" + "strings" + + twitter "github.com/g8rswimmer/go-twitter/v2" + "github.com/nextdotid/proof_server/config" + "golang.org/x/xerrors" +) + +type APIResponse struct { + User struct { + ID string `json:"user_id"` + ScreenName string `json:"screen_name"` + } `json:"user"` + Text string `json:"text"` +} + +var ( + twitterClient *twitter.Client +) + +type authorize struct { + Token string +} + +func (a authorize) Add(req *http.Request) { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", a.Token)) +} + +func initTwitterClient() { + if twitterClient != nil { + return + } + twitterClient = &twitter.Client{ + Authorizer: authorize{ + Token: config.C.Platform.Twitter.OauthToken, + }, + Client: http.DefaultClient, + Host: "https://api.twitter.com", + } +} + +// Fetch tweet using twitter OAuth2.0 API. +// FIXME: should be switched to guest OAuth token solution. +func fetchPostWithAPI(id string, maxRetries int) (*APIResponse, error) { + initTwitterClient() + opts := twitter.TweetLookupOpts{ + Expansions: []twitter.Expansion{twitter.ExpansionEntitiesMentionsUserName, twitter.ExpansionAuthorID}, + TweetFields: []twitter.TweetField{twitter.TweetFieldText, twitter.TweetFieldCreatedAt, twitter.TweetFieldEntities}, + } + result, err := twitterClient.TweetLookup(context.Background(), []string{id}, opts) + if err != nil { + return nil, xerrors.Errorf("error when retriving tweet: %w", err) + } + tweet := result.Raw.Tweets[0] + if tweet == nil { + return nil, xerrors.Errorf("tweet not found: %s", id) + } + + response := APIResponse{ + Text: tweet.Text, + } + response.User.ID = tweet.AuthorID + userName, err := fetchUserName(tweet.AuthorID) + if err != nil { + return nil, err + } + response.User.ScreenName = userName + + return &response, nil +} + +func fetchUserName(userID string) (userName string, err error) { + initTwitterClient() + opts := twitter.UserLookupOpts{ + UserFields: []twitter.UserField{twitter.UserFieldUserName}, + } + result, err := twitterClient.UserLookup(context.Background(), []string{userID}, opts) + if err != nil { + return "", xerrors.Errorf("error when fetching twitter username: %w", err) + } + users := result.Raw.UserDictionaries() + user, ok := users[userID] + if !ok { + return "", xerrors.Errorf("error when fetching twitter username: user not found for ID %s", userID) + } + return strings.ToLower(user.User.UserName), nil +} + +// func fetchPostWithAPI(id string, maxRetries int) (tweet *APIResponse, err error) { +// const RETRY_AFTER = time.Second +// ctx := context.Background() +// if CurrentTokenList == nil { +// CurrentTokenList, err = GetTokenListFromS3(ctx) +// if err != nil { +// return nil, xerrors.Errorf("fetchPostWithAPI: %w", err) +// } +// if CurrentTokenList == nil { +// return nil, xerrors.Errorf("twitter token list does not exist") +// } +// } +// token := lo.Sample(CurrentTokenList.Tokens) +// if lo.IsEmpty(token.OAuthSecret) || lo.IsEmpty(token.OAuthKey) { +// return nil, xerrors.Errorf("twitter token seems to be empty") +// } + +// // TODO: Use token to query specific tweet with twitter API +// // https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets +// // https://api.twitter.com/1.1/statuses/show.json + +// return nil, nil +// } diff --git a/validator/twitter/api_test.go b/validator/twitter/api_test.go new file mode 100644 index 0000000..b46c659 --- /dev/null +++ b/validator/twitter/api_test.go @@ -0,0 +1,25 @@ +package twitter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_fetchPostWithAPI(t *testing.T) { + t.Run("success", func(t *testing.T) { + tweet, err := fetchPostWithAPI("1652176440396517378", 10) + require.NoError(t, err) + require.Contains(t, tweet.Text, "Sig:") + require.Equal(t, tweet.User.ScreenName, "bgm38") + require.Equal(t, tweet.User.ID, "292254624") + }) +} + +func Test_fetchUserName(t *testing.T) { + t.Run("success", func(t *testing.T) { + userName, err := fetchUserName("292254624") + require.NoError(t, err) + require.Equal(t, "bgm38", userName) + }) +} diff --git a/validator/twitter/syndication_api.go b/validator/twitter/syndication_api.go deleted file mode 100644 index 2ba48ca..0000000 --- a/validator/twitter/syndication_api.go +++ /dev/null @@ -1,174 +0,0 @@ -package twitter - -import ( - "encoding/json" - "io" - "net/http" - "time" - - "github.com/samber/lo" - "golang.org/x/xerrors" -) - -type SyndicationAPIResponse struct { - ID string `json:"id_str"` - User SyndicationAPIUser `json:"user"` - Text string `json:"text"` -} - -type SyndicationAPIUser struct { - ID string `json:"id_str"` - ScreenName string `json:"screen_name"` -} - -// / data.threaded_conversation_with_injections.instructions[0].entries[?].content.itemContent.tweet_results.result.legacy.full_text -// / data.threaded_conversation_with_injections.instructions[0].entries[?].content.itemContent.tweet_results.result.core.user_results.result.legacy.name -// / ?: find entryId = "tweet-TWEETID" -type GraphQLAPIResponse struct { - Data struct { - ThreadedConversationWithInjections struct { - Instructions []struct { - Entries []GraphQLAPIEntry `json:"entries"` - } `json:"instructions"` - } `json:"threaded_conversation_with_injections"` - } `json:"data"` -} - -type GraphQLAPIEntry struct { - EntryID string `json:"entryId"` - Content struct { - ItemContent struct { - TweetResults struct { - Result struct { - Core struct { - UserResults struct { - Result struct { - RestID string `json:"rest_id"` - Legacy struct { - ScreenName string `json:"screen_name"` - } `json:"legacy"` - } `json:"result"` - } `json:"user_results"` - } `json:"core"` - Legacy struct { - FullText string `json:"full_text"` - } `json:"legacy"` - } `json:"result"` - } `json:"tweet_results"` - } `json:"itemContent"` - } `json:"content"` -} - -const ( - GUEST_TOKEN_REQUEST = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" - - QUERY_URL_HEAD = "https://api.twitter.com/graphql/miKSMGb2R1SewIJv2-ablQ/TweetDetail?variables=%7B%22focalTweetId%22%3A%22" - QUERY_URL_TAIL = "%22,%22withBirdwatchNotes%22%3Afalse,%22includePromotedContent%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withVoice%22%3Afalse,%22withV2Timeline%22%3Afalse%7D&features=%7B%22blue_business_profile_image_shape_enabled%22%3Afalse,%22rweb_lists_timeline_redesign_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue,%22responsive_web_graphql_timeline_navigation_enabled%22%3Afalse,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Afalse,%22vibe_api_enabled%22%3Afalse,%22responsive_web_edit_tweet_api_enabled%22%3Afalse,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Afalse,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Afalse,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Afalse,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_rich_text_read_enabled%22%3Afalse,%22longform_notetweets_inline_media_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D" - USER_AGENT = "User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -) - -// something like "1691388211879432192" -var GuestToken string - -func fetchPostWithSyndication(id string, maxRetries int) (tweet *SyndicationAPIResponse, err error) { - const RETRY_AFTER = time.Second - - accumulatedErrors := "" - for retry := 0; retry < maxRetries; retry++ { - if retry != 0 { - time.Sleep(RETRY_AFTER) - } - // Fetching guestToken - err := fetchGuestToken() - if err != nil { - accumulatedErrors += (err.Error() + "; ") - continue - } - - tweet, err := fetchPost(id) - if err != nil { - accumulatedErrors += (err.Error() + "; ") - continue - } - return tweet, nil - } - return nil, xerrors.Errorf("%d retries reached: %s", maxRetries, accumulatedErrors) -} - -func fetchGuestToken() (err error) { - if GuestToken != "" { - return nil - } - req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil) - if err != nil { - return err - } - req.Header.Set("Authorization", GUEST_TOKEN_REQUEST) - req.Header.Set("User-Agent", USER_AGENT) - - resp, err := new(http.Client).Do(req) - if err != nil { - return err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - var guestTokenResponse struct { - GuestToken string `json:"guest_token"` - } - err = json.Unmarshal(body, &guestTokenResponse) - if err != nil { - return err - } - if guestTokenResponse.GuestToken == "" { - return xerrors.Errorf("Guest token is empty") - } - - GuestToken = guestTokenResponse.GuestToken - - return nil -} - -func fetchPost(postID string) (post *SyndicationAPIResponse, err error) { - req, err := http.NewRequest("GET", QUERY_URL_HEAD+postID+QUERY_URL_TAIL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", GUEST_TOKEN_REQUEST) - req.Header.Set("User-Agent", USER_AGENT) - req.Header.Set("x-guest-token", GuestToken) - - resp, err := new(http.Client).Do(req) - if err != nil { - return nil, err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - response := new(GraphQLAPIResponse) - err = json.Unmarshal(body, response) - if err != nil { - return nil, err - } - if len(response.Data.ThreadedConversationWithInjections.Instructions) == 0 { - return nil, xerrors.Errorf("No instructions found in response") - } - instruction := response.Data.ThreadedConversationWithInjections.Instructions[0] - entry, found := lo.Find(instruction.Entries, func(entry GraphQLAPIEntry) bool { - return entry.EntryID == ("tweet-" + postID) - }) - if !found { - return nil, xerrors.Errorf("Tweet specified in ProofLocation is not found in API response") - } - - return &SyndicationAPIResponse{ - ID: postID, - User: SyndicationAPIUser{ - ID: entry.Content.ItemContent.TweetResults.Result.Core.UserResults.Result.RestID, - ScreenName: entry.Content.ItemContent.TweetResults.Result.Core.UserResults.Result.Legacy.ScreenName, - }, - Text: entry.Content.ItemContent.TweetResults.Result.Legacy.FullText, - }, nil -} diff --git a/validator/twitter/syndication_api_test.go b/validator/twitter/syndication_api_test.go deleted file mode 100644 index 247c8da..0000000 --- a/validator/twitter/syndication_api_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package twitter - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_SyndicationAPI(t *testing.T) { - t.Run("success", func(t *testing.T) { - // Read /tmp/result_without_timeline.json file and deserialize it into GraphQLResponse - // Then compare it with expected GraphQLResponse - postID := "1687007065032814593" - result, err := fetchPostWithSyndication(postID, 1) - require.NoError(t, err) - require.Equal(t, postID, result.ID) - require.Equal(t, "bgm38", result.User.ScreenName) - require.Equal(t, "292254624", result.User.ID) - }) -} diff --git a/validator/twitter/twitter.go b/validator/twitter/twitter.go index ab935cc..9d27628 100644 --- a/validator/twitter/twitter.go +++ b/validator/twitter/twitter.go @@ -104,12 +104,12 @@ func (twitter *Twitter) Validate() (err error) { // return xerrors.Errorf("fetching tweet with headless browser: %w", err) // } - tweet, err := fetchPostWithSyndication(fmt.Sprint(tweetID), 3) + tweet, err := fetchPostWithAPI(fmt.Sprint(tweetID), 3) if err != nil { return xerrors.Errorf("fetching tweet with syndication API: %w", err) } - if twitter.Identity != strings.ToLower(tweet.User.ScreenName) { - return xerrors.Errorf("Tweet is not sent by this account.") + if twitter.Identity != tweet.User.ScreenName { + return xerrors.Errorf("tweet is not sent by this account.") } twitter.Text = tweet.Text twitter.AltID = tweet.User.ID