xurl webhooks

README.md

@@ -113,6 +113,33 @@ You can also force streaming mode for any endpoint using the `--stream` or `-s`
 xurl -s /2/users/me
 ```
 
+### Temporary Webhook Setup
+
+`xurl` can help you quickly set up a temporary webhook URL to receive events from the X API. This is useful for development and testing.
+
+1.  **Start the local webhook server with ngrok:**
+
+    Run the `webhook start` command. This will start a local server and use ngrok to create a public URL that forwards to your local server. You will be prompted for your ngrok authtoken if it's not already configured via the `NGROK_AUTHTOKEN` environment variable.
+
+    ```bash
+    xurl webhook start
+    # Or with a specific port and output file for POST bodies
+    xurl webhook start -p 8081 -o webhook_events.log
+    ```
+
+    The command will output an ngrok URL (e.g., `https://your-unique-id.ngrok-free.app/webhook`). Note this URL.
+
+2.  **Register the webhook with the X API:**
+
+    Use the ngrok URL obtained in the previous step to register your webhook. You'll typically use app authentication for this.
+
+    ```bash
+    # Replace https://your-ngrok-url.ngrok-free.app/webhook with the actual URL from the previous step
+    xurl --auth app /2/webhooks -d '{"url": "<your ngrok url>"}' -X POST
+    ```
+
+    Your local `xurl webhook start` server will then handle the CRC handshake from Twitter and log incoming POST events (and write them to a file if `-o` was used).
+
 ### Media Upload
 
 The tool supports uploading media files to the X API using the chunked upload process.

cli/root.go

@@ -88,6 +88,7 @@ Examples:
 	rootCmd.AddCommand(CreateAuthCommand(auth))
 	rootCmd.AddCommand(CreateMediaCommand(auth))
 	rootCmd.AddCommand(CreateVersionCommand())
+	rootCmd.AddCommand(CreateWebhookCommand(auth))
 
 	return rootCmd
 }

cli/webhook.go

@@ -0,0 +1,193 @@
+package cli
+
+import (
+	"bufio"
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/spf13/cobra"
+	"github.com/tidwall/pretty"
+	"golang.ngrok.com/ngrok"
+	"golang.ngrok.com/ngrok/config"
+	"xurl/auth"
+)
+
+var webhookPort int
+var outputFileName string // To store the output file name from the flag
+var quietMode bool // To store the quiet flag state
+var prettyMode bool // To store the pretty-print flag state
+
+// CreateWebhookCommand creates the webhook command and its subcommands.
+func CreateWebhookCommand(authInstance *auth.Auth) *cobra.Command {
+	webhookCmd := &cobra.Command{
+		Use:   "webhook",
+		Short: "Manage webhooks for the X API",
+		Long:  `Manages X API webhooks. Currently supports starting a local server with an ngrok tunnel to handle CRC checks.`,
+	}
+
+	webhookStartCmd := &cobra.Command{
+		Use:   "start",
+		Short: "Start a local webhook server with an ngrok tunnel",
+		Long:  `Starts a local HTTP server and an ngrok tunnel to listen for X API webhook events, including CRC checks. POST request bodies can be saved to a file using the -o flag. Use -q for quieter console logging of POST events. Use -p to pretty-print JSON POST bodies in the console.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			color.Cyan("Starting webhook server with ngrok...")
+
+			if authInstance == nil || authInstance.TokenStore == nil {
+				color.Red("Error: Authentication module not initialized properly.")
+				os.Exit(1)
+			}
+
+			oauth1Token := authInstance.TokenStore.GetOAuth1Tokens()
+			if oauth1Token == nil || oauth1Token.OAuth1 == nil || oauth1Token.OAuth1.ConsumerSecret == "" {
+				color.Red("Error: OAuth 1.0a consumer secret not found. Please configure OAuth 1.0a credentials using 'xurl auth oauth1'.")
+				os.Exit(1)
+			}
+			consumerSecret := oauth1Token.OAuth1.ConsumerSecret
+
+			// Handle output file if -o flag is used
+			var outputFile *os.File
+			var errOpenFile error
+			if outputFileName != "" {
+				outputFile, errOpenFile = os.OpenFile(outputFileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+				if errOpenFile != nil {
+					color.Red("Error opening output file %s: %v", outputFileName, errOpenFile)
+					os.Exit(1)
+				}
+				defer outputFile.Close()
+				color.Green("Logging POST request bodies to: %s", outputFileName)
+			}
+
+			// Prompt for ngrok authtoken
+			color.Yellow("Enter your ngrok authtoken (leave empty to try NGROK_AUTHTOKEN env var): ")
+			reader := bufio.NewReader(os.Stdin)
+			ngrokAuthToken, _ := reader.ReadString('\n')
+			ngrokAuthToken = strings.TrimSpace(ngrokAuthToken)
+
+			ctx := context.Background()
+			var tunnelOpts []ngrok.ConnectOption
+			if ngrokAuthToken != "" {
+				tunnelOpts = append(tunnelOpts, ngrok.WithAuthtoken(ngrokAuthToken))
+			} else {
+				color.Cyan("Attempting to use NGROK_AUTHTOKEN environment variable for ngrok authentication.")
+				tunnelOpts = append(tunnelOpts, ngrok.WithAuthtokenFromEnv()) // Fallback to env
+			}
+
+			forwardToAddr := fmt.Sprintf("localhost:%d", webhookPort)
+			color.Cyan("Configuring ngrok to forward to local port: %s", color.MagentaString("%d", webhookPort))
+
+			ngrokListener, err := ngrok.Listen(ctx,
+				config.HTTPEndpoint(
+					config.WithForwardsTo(forwardToAddr),
+				),
+				tunnelOpts...,
+			)
+			if err != nil {
+				color.Red("Error starting ngrok tunnel: %v", err)
+				os.Exit(1)
+			}
+			defer ngrokListener.Close()
+
+			color.Green("Ngrok tunnel established!")
+			fmt.Printf("  Forwarding URL: %s -> %s\n", color.HiGreenString(ngrokListener.URL()), color.MagentaString(forwardToAddr))
+			color.Yellow("Use this URL for your X API webhook registration: %s/webhook", color.HiGreenString(ngrokListener.URL()))
+
+			http.HandleFunc("/webhook", func(w http.ResponseWriter, r *http.Request) {
+				if r.Method == http.MethodGet {
+					crcToken := r.URL.Query().Get("crc_token")
+					if crcToken == "" {
+						http.Error(w, "Error: crc_token missing from request", http.StatusBadRequest)
+						log.Printf("[WARN] Received GET /webhook without crc_token")
+						return
+					}
+					log.Printf("[INFO] Received GET %s%s with crc_token: %s", color.BlueString(r.Host), color.BlueString(r.URL.Path), color.YellowString(crcToken))
+
+					mac := hmac.New(sha256.New, []byte(consumerSecret))
+					mac.Write([]byte(crcToken))
+					hashedToken := mac.Sum(nil)
+					encodedToken := base64.StdEncoding.EncodeToString(hashedToken)
+
+					response := map[string]string{
+						"response_token": "sha256=" + encodedToken,
+					}
+					w.Header().Set("Content-Type", "application/json")
+					json.NewEncoder(w).Encode(response)
+					log.Printf("[INFO] Responded to CRC check with token: %s", color.GreenString(response["response_token"]))
+
+				} else if r.Method == http.MethodPost {
+					bodyBytes, err := io.ReadAll(r.Body)
+					if err != nil {
+						http.Error(w, "Error reading request body", http.StatusInternalServerError)
+						log.Printf("[ERROR] Error reading POST body: %v", err)
+						return
+					}
+					defer r.Body.Close()
+
+					if quietMode {
+						log.Printf("[INFO] Received POST %s%s event (quiet mode).", color.BlueString(r.Host), color.BlueString(r.URL.Path))
+					} else {
+						log.Printf("[INFO] Received POST %s%s event:", color.BlueString(r.Host), color.BlueString(r.URL.Path))
+						if prettyMode {
+							// Attempt to pretty-print if it's JSON
+							var jsonData interface{}
+							if json.Unmarshal(bodyBytes, &jsonData) == nil {
+								prettyColored := pretty.Color(pretty.Pretty(bodyBytes), pretty.TerminalStyle)
+								log.Printf("[DATA] Body:\n%s", string(prettyColored))
+							} else {
+								// Not valid JSON or some other error, print as raw string
+								log.Printf("[DATA] Body (raw, not valid JSON for pretty print):\n%s", string(bodyBytes))
+							}
+						} else {
+							log.Printf("[DATA] Body: %s", string(bodyBytes))
+						}
+					}
+
+					// Write to output file if specified
+					if outputFile != nil {
+						if _, err := outputFile.Write(bodyBytes); err != nil {
+							log.Printf("[ERROR] Error writing POST body to output file %s: %v", outputFileName, err)
+						} else {
+							// Add a separator for readability
+							if _, err := outputFile.WriteString("\n--------------------\n"); err != nil {
+								log.Printf("[ERROR] Error writing separator to output file %s: %v", outputFileName, err)
+							}
+							log.Printf("[INFO] POST body written to %s", color.GreenString(outputFileName))
+						}
+					}
+
+					w.WriteHeader(http.StatusOK)
+				} else {
+					http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+				}
+			})
+
+			color.Cyan("Starting local HTTP server to handle requests from ngrok tunnel (forwarded from %s)...", color.HiGreenString(ngrokListener.URL()))
+			if err := http.Serve(ngrokListener, nil); err != nil {
+				if err != http.ErrServerClosed {
+					color.Red("HTTP server error: %v", err)
+					os.Exit(1)
+				} else {
+					color.Yellow("HTTP server closed gracefully.")
+				}
+			}
+			color.Yellow("Webhook server and ngrok tunnel shut down.")
+		},
+	}
+
+	webhookStartCmd.Flags().IntVarP(&webhookPort, "port", "p", 8080, "Local port for the webhook server to listen on (ngrok will forward to this port)")
+	webhookStartCmd.Flags().StringVarP(&outputFileName, "output", "o", "", "File to write incoming POST request bodies to")
+	webhookStartCmd.Flags().BoolVarP(&quietMode, "quiet", "q", false, "Enable quiet mode (logs only that a POST event was received, not the full body to console)")
+	webhookStartCmd.Flags().BoolVarP(&prettyMode, "pretty", "P", false, "Pretty-print JSON POST bodies in console output (ignored if -q is used)")
+
+	webhookCmd.AddCommand(webhookStartCmd)
+	return webhookCmd
+} 

go.mod

@@ -12,15 +12,26 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible // indirect
+	github.com/inconshreveable/log15/v3 v3.0.0-testing.5 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	golang.org/x/net v0.22.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.ngrok.com/muxado/v2 v2.0.1 // indirect
+	golang.ngrok.com/ngrok v1.13.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -3,15 +3,23 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
 github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
+github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible h1:VryeOTiaZfAzwx8xBcID1KlJCeoWSIpsNbSk+/D2LNk=
+github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o=
+github.com/inconshreveable/log15/v3 v3.0.0-testing.5 h1:h4e0f3kjgg+RJBlKOabrohjHe47D3bbAB9BgMrc3DYA=
+github.com/inconshreveable/log15/v3 v3.0.0-testing.5/go.mod h1:3GQg1SVrLoWGfRv/kAZMsdyU5cp8eFc1P3cw+Wwku94=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -28,17 +36,33 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+golang.ngrok.com/muxado/v2 v2.0.1 h1:jM9i6Pom6GGmnPrHKNR6OJRrUoHFkSZlJ3/S0zqdVpY=
+golang.ngrok.com/muxado/v2 v2.0.1/go.mod h1:wzxJYX4xiAtmwumzL+QsukVwFRXmPNv86vB8RPpOxyM=
+golang.ngrok.com/ngrok v1.13.0 h1:6SeOS+DAeIaHlkDmNH5waFHv0xjlavOV3wml0Z59/8k=
+golang.ngrok.com/ngrok v1.13.0/go.mod h1:BKOMdoZXfD4w6o3EtE7Cu9TVbaUWBqptrZRWnVcAuI4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
 golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
 golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
+golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -49,7 +73,11 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=