Fix large attachment handling with string keys and custom content_ids (#537)

Author: AaronDDM

.rubocop.yml

@@ -1,11 +1,12 @@
 inherit_from: .rubocop_todo.yml
 
-require:
+plugins:
   - rubocop-rspec
   - rubocop-capybara
 
 AllCops:
   TargetRubyVersion: 3.0
+  NewCops: disable
   DisplayCopNames: true
   DisplayStyleGuide: true
   Exclude:
@@ -57,5 +58,18 @@ RSpec/MultipleExpectations:
 RSpec/ExampleLength:
   Enabled: false
 
-RSpec/FilePath:
+RSpec/SpecFilePathFormat:
+  Enabled: false
+RSpec/SpecFilePathSuffix:
+  Enabled: false
+
+RSpec/VerifiedDoubleReference:
+  Enabled: false
+RSpec/BeEq:
+  Enabled: false
+RSpec/IdenticalEqualityAssertion:
+  Enabled: false
+RSpec/NoExpectationExample:
+  Enabled: false
+RSpec/ReceiveMessages:
   Enabled: false

examples/README.md

@@ -12,6 +12,7 @@ examples/
 ├── messages/                 # Message-related examples
 │   ├── message_fields_example.rb     # Example of using new message fields functionality
 │   ├── file_upload_example.rb        # Example of file upload functionality with HTTParty migration
+│   ├── send_streaming_attachments_example.rb  # Sending attachments from a stream (no local file)
 │   └── send_message_example.rb       # Example of basic message sending functionality
 └── notetaker/               # Standalone Notetaker examples
     ├── README.md            # Notetaker-specific documentation
@@ -68,6 +69,17 @@ Before running any example, make sure to:
   export NYLAS_TEST_EMAIL="test@example.com"  # Email address to send test messages to
   ```
 
+- `messages/send_streaming_attachments_example.rb`: Sending attachments from a stream (no local file on disk), including:
+  - Passing string content from an IO/stream instead of a file path
+  - Small attachments (<3MB) via JSON base64
+  - Large attachments (>3MB) via multipart: `LARGE_ATTACHMENT=1 ruby ...`
+
+  Additional environment variables needed:
+  ```bash
+  export NYLAS_GRANT_ID="your_grant_id"
+  export NYLAS_TEST_EMAIL="test@example.com"
+  ```
+
 - `messages/send_message_example.rb`: Demonstrates basic message sending functionality, including:
   - Sending simple text messages
   - Handling multiple recipients (TO, CC, BCC)

examples/messages/send_streaming_attachments_example.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example: Sending attachments from a stream (no local file on disk)
+#
+# When content comes from a stream (network, database, etc.), read it into a
+# string and pass it to the SDK. You do not need a local file path.
+#
+#   stream = some_source.read  # IO, StringIO, HTTP response body, etc.
+#   attachment = { filename: "doc.pdf", content_type: "application/pdf", size: stream.bytesize, content: stream }
+#
+# Environment variables:
+#   NYLAS_API_KEY     - Your Nylas API key
+#   NYLAS_GRANT_ID    - Grant ID (connected account)
+#   NYLAS_TEST_EMAIL  - Recipient email
+#
+# Optional: NYLAS_API_URI (default: https://api.us.nylas.com)
+# Optional: LARGE_ATTACHMENT=1 for >3MB (multipart path)
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "nylas"
+
+def load_env
+  env_file = File.expand_path("../.env", __dir__)
+  return unless File.exist?(env_file)
+
+  File.readlines(env_file).each do |line|
+    line = line.strip
+    next if line.empty? || line.start_with?("#")
+
+    key, value = line.split("=", 2)
+    ENV[key] = value&.gsub(/\A['"]|['"]\z/, "") if key && value
+  end
+end
+
+def attachment_from_stream(io, filename:, content_type:)
+  content = io.read
+  io.close if io.respond_to?(:close)
+
+  {
+    filename: filename,
+    content_type: content_type,
+    size: content.bytesize,
+    content: content
+  }
+end
+
+def main
+  load_env
+
+  api_key = ENV["NYLAS_API_KEY"]
+  grant_id = ENV["NYLAS_GRANT_ID"]
+  recipient = ENV["NYLAS_TEST_EMAIL"]
+
+  raise "Set NYLAS_API_KEY, NYLAS_GRANT_ID, NYLAS_TEST_EMAIL" unless api_key && grant_id && recipient
+
+  nylas = Nylas::Client.new(
+    api_key: api_key,
+    api_uri: ENV["NYLAS_API_URI"] || "https://api.us.nylas.com"
+  )
+
+  use_large = ENV["LARGE_ATTACHMENT"] == "1"
+
+  if use_large
+    stream = StringIO.new("%PDF-1.4\n" + ("x" * (4 * 1024 * 1024 - 32)))
+    attachment = attachment_from_stream(stream, filename: "report.pdf", content_type: "application/pdf")
+    puts "Using large attachment (>3MB) - multipart form-data path"
+  else
+    stream = StringIO.new("%PDF-1.4 simulated content " + ("x" * 1024))
+    attachment = attachment_from_stream(stream, filename: "report.pdf", content_type: "application/pdf")
+    puts "Using small attachment (<3MB) - JSON base64 path"
+  end
+
+  puts "Sending email with streamed attachment..."
+  puts "  Attachment: #{attachment[:filename]} (#{attachment[:size]} bytes)"
+  puts "  No local file - content from stream"
+
+  response, request_id = nylas.messages.send(
+    identifier: grant_id,
+    request_body: {
+      subject: "Report",
+      body: "Attached document from stream.",
+      to: [{ email: recipient }],
+      attachments: [attachment]
+    }
+  )
+
+  puts "Sent. Message ID: #{response[:id]}, Request ID: #{request_id}"
+end
+
+main if __FILE__ == $PROGRAM_NAME

gem_config.rb

@@ -38,7 +38,7 @@ def self.dev_dependencies
     [["bundler", ">= 1.3.0"],
      ["yard", "~> 0.9.34"],
      ["rubocop", "~> 1.51"],
-     ["rubocop-rspec", "~> 2.22"],
+     ["rubocop-rspec", "~> 3.5"],
      ["rubocop-capybara", "~> 2.20"]] + testing_and_debugging_dependencies
   end
 

lib/nylas/handler/http_client.rb

@@ -2,6 +2,7 @@
 
 require "httparty"
 require "net/http"
+require "net/http/post/multipart"
 
 require_relative "../errors"
 require_relative "../version"
@@ -135,40 +136,104 @@ def parse_response(response)
 
     private
 
-    # Sends a request to the Nylas REST API using HTTParty.
+    # Sends a request to the Nylas REST API using HTTParty or Net::HTTP for multipart.
+    # Multipart requests use Net::HTTP::Post::Multipart (multipart-post gem) because
+    # HTTParty's multipart handling produces malformed requests that the Nylas API rejects.
     #
     # @param method [Symbol] HTTP method for the API call. Either :get, :post, :delete, or :patch.
     # @param url [String] URL for the API call.
     # @param headers [Hash] HTTP headers to include in the payload.
     # @param payload [String, Hash] Body to send with the request.
     # @param timeout [Hash] Timeout value to send with the request.
     def httparty_execute(method:, url:, headers:, payload:, timeout:)
-      options = {
-        headers: headers,
-        timeout: timeout
-      }
-
-      # Handle multipart uploads
-      if payload.is_a?(Hash) && file_upload?(payload)
-        options[:multipart] = true
-        options[:body] = prepare_multipart_payload(payload)
-      elsif payload
-        options[:body] = payload
+      if method == :post && payload.is_a?(Hash) && file_upload?(payload)
+        response = execute_multipart_request(url: url, headers: headers, payload: payload, timeout: timeout)
+      else
+        options = { headers: headers, timeout: timeout }
+        options[:body] = payload if payload
+        response = HTTParty.send(method, url, options)
       end
 
-      response = HTTParty.send(method, url, options)
-
-      # Create a compatible response object that mimics RestClient::Response
       result = create_response_wrapper(response)
 
-      # Call the block with the response in the same format as rest-client
       if block_given?
         yield response, nil, result
       else
         response
       end
     end
 
+    # Executes multipart POST using Net::HTTP::Post::Multipart (fixes issue #538).
+    # HTTParty's multipart produces malformed requests; multipart-post/UploadIO works correctly.
+    def execute_multipart_request(url:, headers:, payload:, timeout:)
+      uri = URI.parse(url)
+      params = build_multipart_params(payload)
+
+      req = Net::HTTP::Post::Multipart.new(uri.path, params)
+      headers.each { |key, value| req[key] = value }
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == "https")
+      http.read_timeout = timeout
+      http.open_timeout = timeout
+
+      response = http.request(req)
+
+      create_httparty_like_response(response)
+    end
+
+    # Build params hash for Net::HTTP::Post::Multipart with UploadIO for file fields.
+    def build_multipart_params(payload)
+      params = {}
+      payload.each do |key, value|
+        params[key] = if key.is_a?(String) && key != "message" && file_like_value?(value)
+                        value_to_upload_io(value)
+                      else
+                        value.to_s
+                      end
+      end
+      params
+    end
+
+    def file_like_value?(value)
+      return true if value.respond_to?(:read) && (value.is_a?(File) ? !value.closed? : true)
+      if value.is_a?(String) && (value.respond_to?(:original_filename) || value.respond_to?(:content_type))
+        return true
+      end
+
+      false
+    end
+
+    # Convert File, String, or StringIO to UploadIO for multipart-post.
+    def value_to_upload_io(value)
+      content_type = value.respond_to?(:content_type) ? value.content_type : "application/octet-stream"
+      filename = value.respond_to?(:original_filename) ? value.original_filename : "file.bin"
+
+      io = if value.respond_to?(:read) && value.respond_to?(:rewind)
+             value.rewind if value.respond_to?(:rewind)
+             value
+           else
+             require "stringio"
+             content = value.to_s
+             content = content.dup.force_encoding(Encoding::ASCII_8BIT) if content.is_a?(String)
+             StringIO.new(content)
+           end
+
+      UploadIO.new(io, content_type, filename)
+    end
+
+    # Create response object compatible with HTTParty::Response interface.
+    def create_httparty_like_response(net_http_response)
+      headers = net_http_response.to_hash
+      headers = headers.transform_values { |v| v.is_a?(Array) && v.one? ? v.first : v }
+
+      OpenStruct.new(
+        body: net_http_response.body,
+        code: net_http_response.code.to_i,
+        headers: headers
+      )
+    end
+
     # Create a response wrapper that mimics RestClient::Response.code behavior
     def create_response_wrapper(response)
       OpenStruct.new(code: response.code)
@@ -188,10 +253,20 @@ def file_upload?(payload)
       # Check if payload was prepared by FileUtils.build_form_request for multipart uploads
       # This handles binary content attachments that are strings with added singleton methods
       has_message_field = payload.key?("message") && payload["message"].is_a?(String)
-      has_attachment_fields = payload.keys.any? { |key| key.is_a?(String) && key.match?(/^file\d+$/) }
 
-      # If we have both a "message" field and "file{N}" fields, this indicates
-      # the payload was prepared by FileUtils.build_form_request for multipart upload
+      # Check for attachment fields - these can have custom content_id values (not just "file{N}")
+      # FileUtils.build_form_request creates entries with string values that have singleton methods
+      # like original_filename and content_type defined on them
+      has_attachment_fields = payload.any? do |key, value|
+        next false unless key.is_a?(String) && key != "message"
+
+        # Check if the value is a string with attachment-like singleton methods
+        # (original_filename or content_type), which indicates it's a file content
+        value.is_a?(String) && (value.respond_to?(:original_filename) || value.respond_to?(:content_type))
+      end
+
+      # If we have both a "message" field and attachment fields with file metadata,
+      # this indicates the payload was prepared by FileUtils.build_form_request
       has_message_field && has_attachment_fields
     end
 

lib/nylas/resources/webhooks.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "cgi"
 require_relative "resource"
 require_relative "../handler/api_operations"
 
@@ -115,14 +114,14 @@ def ip_addresses
     # @return [String] The challenge parameter
     def self.extract_challenge_parameter(url)
       url_object = URI.parse(url)
-      query = CGI.parse(url_object.query || "")
+      params = URI.decode_www_form(url_object.query || "")
+      challenge_pair = params.find { |k, _| k == "challenge" }
 
-      challenge_parameter = query["challenge"]
-      if challenge_parameter.nil? || challenge_parameter.empty? || challenge_parameter.first.nil?
+      if challenge_pair.nil? || challenge_pair.last.to_s.empty?
         raise "Invalid URL or no challenge parameter found."
       end
 
-      challenge_parameter.first
+      challenge_pair.last
     end
   end
 end

lib/nylas/utils/file_utils.rb

@@ -28,12 +28,13 @@ def self.build_form_request(request_body)
 
       attachments.each_with_index do |attachment, index|
         file = attachment[:content] || attachment["content"]
+        file_path = attachment[:file_path] || attachment["file_path"]
         if file.respond_to?(:closed?) && file.closed?
-          unless attachment[:file_path]
+          unless file_path
             raise ArgumentError, "The file at index #{index} is closed and no file_path was provided."
           end
 
-          file = File.open(attachment[:file_path], "rb")
+          file = File.open(file_path, "rb")
         end
 
         # Setting original filename and content type if available. See rest-client#lib/restclient/payload.rb
@@ -87,7 +88,9 @@ def self.handle_message_payload(request_body)
 
       # Use form data only if the attachment size is greater than 3mb
       attachments = payload[:attachments]
-      attachment_size = attachments&.sum { |attachment| attachment[:size] || 0 } || 0
+      # Support both string and symbol keys for attachment size to handle
+      # user-provided hashes that may use either key type
+      attachment_size = attachments&.sum { |attachment| attachment[:size] || attachment["size"] || 0 } || 0
 
       # Handle the attachment encoding depending on the size
       if attachment_size >= FORM_DATA_ATTACHMENT_SIZE

nylas.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "base64"
   gem.add_runtime_dependency "httparty", "~> 0.21"
   gem.add_runtime_dependency "mime-types", "~> 3.5", ">= 3.5.1"
+  gem.add_runtime_dependency "multipart-post", "~> 2.0"
   gem.add_runtime_dependency "ostruct", "~> 0.6"
   gem.add_runtime_dependency "yajl-ruby", "~> 1.4.3", ">= 1.2.1"