| Name | Total Lines | Lines of Code | Total Coverage | Code Coverage |
|---|---|---|---|---|
| lib/buildr/core/transports.rb | 565 | 357 | 17.35%
|
18.77%
|
Code reported as executed by Ruby looks like this...and this: this line is also marked as covered.Lines considered as run by rcov, but not reported by Ruby, look like this,and this: these lines were inferred by rcov (using simple heuristics).Finally, here's a line marked as not executed.
1 # Licensed to the Apache Software Foundation (ASF) under one or more |
2 # contributor license agreements. See the NOTICE file distributed with this |
3 # work for additional information regarding copyright ownership. The ASF |
4 # licenses this file to you under the Apache License, Version 2.0 (the |
5 # "License"); you may not use this file except in compliance with the License. |
6 # You may obtain a copy of the License at |
7 # |
8 # http://www.apache.org/licenses/LICENSE-2.0 |
9 # |
10 # Unless required by applicable law or agreed to in writing, software |
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
13 # License for the specific language governing permissions and limitations under |
14 # the License. |
15 |
16 |
17 require 'uri' |
18 require 'net/http' |
19 # PATCH: On Windows, Net::SSH 2.0.2 attempts to load the Pageant DLLs which break on JRuby. |
20 $LOADED_FEATURES << 'net/ssh/authentication/pageant.rb' if RUBY_PLATFORM =~ /java/ |
21 gem 'net-ssh' ; Net.autoload :SSH, 'net/ssh' |
22 gem 'net-sftp' ; Net.autoload :SFTP, 'net/sftp' |
23 autoload :CGI, 'cgi' |
24 require 'digest/md5' |
25 require 'digest/sha1' |
26 require 'stringio' |
27 autoload :ProgressBar, 'buildr/core/progressbar' |
28 |
29 |
30 # Not quite open-uri, but similar. Provides read and write methods for the resource represented by the URI. |
31 # Currently supports reads for URI::HTTP and writes for URI::SFTP. Also provides convenience methods for |
32 # downloads and uploads. |
33 module URI |
34 |
35 # Raised when trying to read/download a resource that doesn't exist. |
36 class NotFoundError < RuntimeError |
37 end |
38 |
39 # How many bytes to read/write at once. Do not change without checking BUILDR-214 first. |
40 RW_CHUNK_SIZE = 128 * 1024 #:nodoc: |
41 |
42 class << self |
43 |
44 # :call-seq: |
45 # read(uri, options?) => content |
46 # read(uri, options?) { |chunk| ... } |
47 # |
48 # Reads from the resource behind this URI. The first form returns the content of the resource, |
49 # the second form yields to the block with each chunk of content (usually more than one). |
50 # |
51 # For example: |
52 # File.open 'image.jpg', 'w' do |file| |
53 # URI.read('http://example.com/image.jpg') { |chunk| file.write chunk } |
54 # end |
55 # Shorter version: |
56 # File.open('image.jpg', 'w') { |file| file.write URI.read('http://example.com/image.jpg') } |
57 # |
58 # Supported options: |
59 # * :modified -- Only download if file modified since this timestamp. Returns nil if not modified. |
60 # * :progress -- Show the progress bar while reading. |
61 def read(uri, options = nil, &block) |
62 uri = URI.parse(uri.to_s) unless URI === uri |
63 uri.read options, &block |
64 end |
65 |
66 # :call-seq: |
67 # download(uri, target, options?) |
68 # |
69 # Downloads the resource to the target. |
70 # |
71 # The target may be a file name (string or task), in which case the file is created from the resource. |
72 # The target may also be any object that responds to +write+, e.g. File, StringIO, Pipe. |
73 # |
74 # Use the progress bar when running in verbose mode. |
75 def download(uri, target, options = nil) |
76 uri = URI.parse(uri.to_s) unless URI === uri |
77 uri.download target, options |
78 end |
79 |
80 # :call-seq: |
81 # write(uri, content, options?) |
82 # write(uri, options?) { |bytes| .. } |
83 # |
84 # Writes to the resource behind the URI. The first form writes the content from a string or an object |
85 # that responds to +read+ and optionally +size+. The second form writes the content by yielding to the |
86 # block. Each yield should return up to the specified number of bytes, the last yield returns nil. |
87 # |
88 # For example: |
89 # File.open 'killer-app.jar', 'rb' do |file| |
90 # write('sftp://localhost/jars/killer-app.jar') { |chunk| file.read(chunk) } |
91 # end |
92 # Or: |
93 # write 'sftp://localhost/jars/killer-app.jar', File.read('killer-app.jar') |
94 # |
95 # Supported options: |
96 # * :progress -- Show the progress bar while reading. |
97 def write(uri, *args, &block) |
98 uri = URI.parse(uri.to_s) unless URI === uri |
99 uri.write *args, &block |
100 end |
101 |
102 # :call-seq: |
103 # upload(uri, source, options?) |
104 # |
105 # Uploads from source to the resource. |
106 # |
107 # The source may be a file name (string or task), in which case the file is uploaded to the resource. |
108 # The source may also be any object that responds to +read+ (and optionally +size+), e.g. File, StringIO, Pipe. |
109 # |
110 # Use the progress bar when running in verbose mode. |
111 def upload(uri, source, options = nil) |
112 uri = URI.parse(uri.to_s) unless URI === uri |
113 uri.upload source, options |
114 end |
115 |
116 end |
117 |
118 class Generic |
119 |
120 # :call-seq: |
121 # read(options?) => content |
122 # read(options?) { |chunk| ... } |
123 # |
124 # Reads from the resource behind this URI. The first form returns the content of the resource, |
125 # the second form yields to the block with each chunk of content (usually more than one). |
126 # |
127 # For options, see URI::read. |
128 def read(options = nil, &block) |
129 fail 'This protocol doesn\'t support reading (yet, how about helping by implementing it?)' |
130 end |
131 |
132 # :call-seq: |
133 # download(target, options?) |
134 # |
135 # Downloads the resource to the target. |
136 # |
137 # The target may be a file name (string or task), in which case the file is created from the resource. |
138 # The target may also be any object that responds to +write+, e.g. File, StringIO, Pipe. |
139 # |
140 # Use the progress bar when running in verbose mode. |
141 def download(target, options = nil) |
142 case target |
143 when Rake::Task |
144 download target.name, options |
145 when String |
146 # If download breaks we end up with a partial file which is |
147 # worse than not having a file at all, so download to temporary |
148 # file and then move over. |
149 modified = File.stat(target).mtime if File.exist?(target) |
150 temp = Tempfile.new(File.basename(target)) |
151 temp.binmode |
152 read({:progress=>verbose}.merge(options || {}).merge(:modified=>modified)) { |chunk| temp.write chunk } |
153 temp.close |
154 mkpath File.dirname(target) |
155 mv temp.path, target |
156 when File |
157 read({:progress=>verbose}.merge(options || {}).merge(:modified=>target.mtime)) { |chunk| target.write chunk } |
158 target.flush |
159 else |
160 raise ArgumentError, 'Expecting a target that is either a file name (string, task) or object that responds to write (file, pipe).' unless target.respond_to?(:write) |
161 read({:progress=>verbose}.merge(options || {})) { |chunk| target.write chunk } |
162 target.flush |
163 end |
164 end |
165 |
166 # :call-seq: |
167 # write(content, options?) |
168 # write(options?) { |bytes| .. } |
169 # |
170 # Writes to the resource behind the URI. The first form writes the content from a string or an object |
171 # that responds to +read+ and optionally +size+. The second form writes the content by yielding to the |
172 # block. Each yield should return up to the specified number of bytes, the last yield returns nil. |
173 # |
174 # For options, see URI::write. |
175 def write(*args, &block) |
176 options = args.pop if Hash === args.last |
177 options ||= {} |
178 if String === args.first |
179 ios = StringIO.new(args.first, 'r') |
180 write(options.merge(:size=>args.first.size)) { |bytes| ios.read(bytes) } |
181 elsif args.first.respond_to?(:read) |
182 size = args.first.size rescue nil |
183 write({:size=>size}.merge(options)) { |bytes| args.first.read(bytes) } |
184 elsif args.empty? && block |
185 write_internal options, &block |
186 else |
187 raise ArgumentError, 'Either give me the content, or pass me a block, otherwise what would I upload?' |
188 end |
189 end |
190 |
191 # :call-seq: |
192 # upload(source, options?) |
193 # |
194 # Uploads from source to the resource. |
195 # |
196 # The source may be a file name (string or task), in which case the file is uploaded to the resource. |
197 # If the source is a directory, uploads all files inside the directory (including nested directories). |
198 # The source may also be any object that responds to +read+ (and optionally +size+), e.g. File, StringIO, Pipe. |
199 # |
200 # Use the progress bar when running in verbose mode. |
201 def upload(source, options = nil) |
202 source = source.name if Rake::Task === source |
203 options ||= {} |
204 if String === source |
205 raise NotFoundError, 'No source file/directory to upload.' unless File.exist?(source) |
206 if File.directory?(source) |
207 Dir.glob("#{source}/**/*").reject { |file| File.directory?(file) }.each do |file| |
208 uri = self + (File.join(self.path, file.sub(source, ''))) |
209 uri.upload file, {:digests=>[]}.merge(options) |
210 end |
211 else |
212 File.open(source, 'rb') { |input| upload input, options } |
213 end |
214 elsif source.respond_to?(:read) |
215 digests = (options[:digests] || [:md5, :sha1]). |
216 inject({}) { |hash, name| hash[name] = Digest.const_get(name.to_s.upcase).new ; hash } |
217 size = source.stat.size rescue nil |
218 write (options).merge(:progress=>verbose && size, :size=>size) do |bytes| |
219 source.read(bytes).tap do |chunk| |
220 digests.values.each { |digest| digest << chunk } if chunk |
221 end |
222 end |
223 digests.each do |key, digest| |
224 self.merge("#{self.path}.#{key}").write digest.hexdigest, |
225 (options).merge(:progress=>false) |
226 end |
227 else |
228 raise ArgumentError, 'Expecting source to be a file name (string, task) or any object that responds to read (file, pipe).' |
229 end |
230 end |
231 |
232 protected |
233 |
234 # :call-seq: |
235 # with_progress_bar(show, file_name, size) { |progress| ... } |
236 # |
237 # Displays a progress bar while executing the block. The first argument must be true for the |
238 # progress bar to show (TTY output also required), as a convenient for selectively using the |
239 # progress bar from a single block. |
240 # |
241 # The second argument provides a filename to display, the third its size in bytes. |
242 # |
243 # The block is yielded with a progress object that implements a single method. |
244 # Call << for each block of bytes down/uploaded. |
245 def with_progress_bar(show, file_name, size, &block) #:nodoc: |
246 options = { :total=>size || 0, :title=>file_name } |
247 options[:hidden] = true unless show |
248 ProgressBar.start options, &block |
249 end |
250 |
251 # :call-seq: |
252 # proxy_uri => URI? |
253 # |
254 # Returns the proxy server to use. Obtains the proxy from the relevant environment variable (e.g. HTTP_PROXY). |
255 # Supports exclusions based on host name and port number from environment variable NO_PROXY. |
256 def proxy_uri |
257 proxy = ENV["#{scheme.upcase}_PROXY"] |
258 proxy = URI.parse(proxy) if String === proxy |
259 excludes = ENV['NO_PROXY'].to_s.split(/\s*,\s*/).compact |
260 excludes = excludes.map { |exclude| exclude =~ /:\d+$/ ? exclude : "#{exclude}:*" } |
261 return proxy unless excludes.any? { |exclude| File.fnmatch(exclude, "#{host}:#{port}") } |
262 end |
263 |
264 def write_internal(options, &block) #:nodoc: |
265 fail 'This protocol doesn\'t support writing (yet, how about helping by implementing it?)' |
266 end |
267 |
268 end |
269 |
270 |
271 class HTTP #:nodoc: |
272 |
273 # See URI::Generic#read |
274 def read(options = nil, &block) |
275 options ||= {} |
276 connect do |http| |
277 trace "Requesting #{self}" |
278 headers = { 'If-Modified-Since' => CGI.rfc1123_date(options[:modified].utc) } if options[:modified] |
279 request = Net::HTTP::Get.new(request_uri.empty? ? '/' : request_uri, headers) |
280 request.basic_auth self.user, self.password if self.user |
281 http.request request do |response| |
282 case response |
283 when Net::HTTPNotModified |
284 # No modification, nothing to do. |
285 trace 'Not modified since last download' |
286 return nil |
287 when Net::HTTPRedirection |
288 # Try to download from the new URI, handle relative redirects. |
289 trace "Redirected to #{response['Location']}" |
290 rself = self + URI.parse(response['Location']) |
291 rself.user, rself.password = self.user, self.password |
292 return rself.read(options, &block) |
293 when Net::HTTPOK |
294 info "Downloading #{self}" |
295 result = nil |
296 with_progress_bar options[:progress], path.split('/').last, response.content_length do |progress| |
297 if block |
298 response.read_body do |chunk| |
299 block.call chunk |
300 progress << chunk |
301 end |
302 else |
303 result = '' |
304 response.read_body do |chunk| |
305 result << chunk |
306 progress << chunk |
307 end |
308 end |
309 end |
310 return result |
311 when Net::HTTPNotFound |
312 raise NotFoundError, "Looking for #{self} and all I got was a 404!" |
313 else |
314 raise RuntimeError, "Failed to download #{self}: #{response.message}" |
315 end |
316 end |
317 end |
318 end |
319 |
320 private |
321 |
322 def write_internal(options, &block) #:nodoc: |
323 options ||= {} |
324 connect do |http| |
325 trace "Uploading to #{path}" |
326 content = StringIO.new |
327 while chunk = yield(RW_CHUNK_SIZE) |
328 content << chunk |
329 end |
330 headers = { 'Content-MD5'=>Digest::MD5.hexdigest(content.string), 'Content-Type'=>'application/octet-stream' } |
331 request = Net::HTTP::Put.new(request_uri.empty? ? '/' : request_uri, headers) |
332 request.basic_auth self.user, self.password if self.user |
333 response = nil |
334 with_progress_bar options[:progress], path.split('/').last, content.size do |progress| |
335 request.content_length = content.size |
336 content.rewind |
337 stream = Object.new |
338 class << stream ; self ;end.send :define_method, :read do |count| |
339 bytes = content.read(count) |
340 progress << bytes if bytes |
341 bytes |
342 end |
343 request.body_stream = stream |
344 response = http.request(request) |
345 end |
346 |
347 case response |
348 when Net::HTTPRedirection |
349 # Try to download from the new URI, handle relative redirects. |
350 trace "Redirected to #{response['Location']}" |
351 content.rewind |
352 return (self + URI.parse(response['location'])).write_internal(options) { |bytes| content.read(bytes) } |
353 when Net::HTTPSuccess |
354 else |
355 raise RuntimeError, "Failed to upload #{self}: #{response.message}" |
356 end |
357 end |
358 end |
359 |
360 def connect |
361 if proxy = proxy_uri |
362 proxy = URI.parse(proxy) if String === proxy |
363 http = Net::HTTP.new(host, port, proxy.host, proxy.port, proxy.user, proxy.password) |
364 else |
365 http = Net::HTTP.new(host, port) |
366 end |
367 if self.instance_of? URI::HTTPS |
368 require 'net/https' |
369 http.use_ssl = true |
370 end |
371 yield http |
372 end |
373 |
374 end |
375 |
376 |
377 class SFTP < Generic #:nodoc: |
378 |
379 DEFAULT_PORT = 22 |
380 COMPONENT = [ :scheme, :userinfo, :host, :port, :path ].freeze |
381 |
382 class << self |
383 # Caching of passwords, so we only need to ask once. |
384 def passwords |
385 @passwords ||= {} |
386 end |
387 end |
388 |
389 def initialize(*arg) |
390 super |
391 end |
392 |
393 def read(options = {}, &block) |
394 # SSH options are based on the username/password from the URI. |
395 ssh_options = { :port=>port, :password=>password }.merge(options[:ssh_options] || {}) |
396 ssh_options[:password] ||= SFTP.passwords[host] |
397 begin |
398 trace "Connecting to #{host}" |
399 if block |
400 result = nil |
401 else |
402 result = '' |
403 block = lambda { |chunk| result << chunk } |
404 end |
405 Net::SFTP.start(host, user, ssh_options) do |sftp| |
406 SFTP.passwords[host] = ssh_options[:password] |
407 trace 'connected' |
408 |
409 with_progress_bar options[:progress] && options[:size], path.split('/').last, options[:size] || 0 do |progress| |
410 trace "Downloading from #{path}" |
411 sftp.file.open(path, 'r') do |file| |
412 while chunk = file.read(RW_CHUNK_SIZE) |
413 block.call chunk |
414 progress << chunk |
415 break if chunk.size < RW_CHUNK_SIZE |
416 end |
417 end |
418 end |
419 end |
420 return result |
421 rescue Net::SSH::AuthenticationFailed=>ex |
422 # Only if running with console, prompt for password. |
423 if !ssh_options[:password] && $stdout.isatty |
424 password = ask("Password for #{host}:") { |q| q.echo = '*' } |
425 ssh_options[:password] = password |
426 retry |
427 end |
428 raise |
429 end |
430 end |
431 |
432 protected |
433 |
434 def write_internal(options, &block) #:nodoc: |
435 # SSH options are based on the username/password from the URI. |
436 ssh_options = { :port=>port, :password=>password }.merge(options[:ssh_options] || {}) |
437 ssh_options[:password] ||= SFTP.passwords[host] |
438 begin |
439 trace "Connecting to #{host}" |
440 Net::SFTP.start(host, user, ssh_options) do |sftp| |
441 SFTP.passwords[host] = ssh_options[:password] |
442 trace 'Connected' |
443 |
444 # To create a path, we need to create all its parent. We use realpath to determine if |
445 # the path already exists, otherwise mkdir fails. |
446 trace "Creating path #{path}" |
447 File.dirname(path).split('/').reject(&:empty?).inject('/') do |base, part| |
448 combined = base + part |
449 sftp.close(sftp.opendir!(combined)) rescue sftp.mkdir! combined, {} |
450 "#{combined}/" |
451 end |
452 |
453 with_progress_bar options[:progress] && options[:size], path.split('/').last, options[:size] || 0 do |progress| |
454 trace "Uploading to #{path}" |
455 sftp.file.open(path, 'w') do |file| |
456 while chunk = yield(RW_CHUNK_SIZE) |
457 file.write chunk |
458 progress << chunk |
459 end |
460 sftp.setstat(path, :permissions => options[:permissions]) if options[:permissions] |
461 end |
462 end |
463 end |
464 rescue Net::SSH::AuthenticationFailed=>ex |
465 # Only if running with console, prompt for password. |
466 if !ssh_options[:password] && $stdout.isatty |
467 password = ask("Password for #{host}:") { |q| q.echo = '*' } |
468 ssh_options[:password] = password |
469 retry |
470 end |
471 raise |
472 end |
473 end |
474 |
475 end |
476 |
477 @@schemes['SFTP'] = SFTP |
478 |
479 |
480 # File URL. Keep in mind that file URLs take the form of <code>file://host/path</code>, although the host |
481 # is not used, so typically all you will see are three backslashes. This methods accept common variants, |
482 # like <code>file:/path</code> but always returns a valid URL. |
483 class FILE < Generic |
484 |
485 COMPONENT = [ :host, :path ].freeze |
486 |
487 def upload(source, options = nil) |
488 super |
489 if File === source then |
490 File.chmod(source.stat.mode, real_path) |
491 end |
492 end |
493 |
494 def initialize(*args) |
495 super |
496 # file:something (opaque) becomes file:///something |
497 if path.nil? |
498 set_path "/#{opaque}" |
499 unless opaque.nil? |
500 set_opaque nil |
501 warn "#{caller[2]}: We'll accept this URL, but just so you know, it needs three slashes, as in: #{to_s}" |
502 end |
503 end |
504 # Sadly, file://something really means file://something/ (something being server) |
505 set_path '/' if path.empty? |
506 |
507 # On windows, file://c:/something is not a valid URL, but people do it anyway, so if we see a drive-as-host, |
508 # we'll just be nice enough to fix it. (URI actually strips the colon here) |
509 if host =~ /^[a-zA-Z]$/ |
510 set_path "/#{host}:#{path}" |
511 set_host nil |
512 end |
513 end |
514 |
515 # See URI::Generic#read |
516 def read(options = nil, &block) |
517 options ||= {} |
518 raise ArgumentError, 'Either you\'re attempting to read a file from another host (which we don\'t support), or you used two slashes by mistake, where you should have file:///<path>.' if host |
519 |
520 path = real_path |
521 # TODO: complain about clunky URLs |
522 raise NotFoundError, "Looking for #{self} and can't find it." unless File.exists?(path) |
523 raise NotFoundError, "Looking for the file #{self}, and it happens to be a directory." if File.directory?(path) |
524 File.open path, 'rb' do |input| |
525 with_progress_bar options[:progress], path.split('/').last, input.stat.size do |progress| |
526 block ? block.call(input.read) : input.read |
527 end |
528 end |
529 end |
530 |
531 def to_s |
532 "file://#{host}#{path}" |
533 end |
534 |
535 # Returns the file system path based that corresponds to the URL path. |
536 # On windows this method strips the leading slash off of the path. |
537 # On all platforms this method unescapes the URL path. |
538 def real_path #:nodoc: |
539 real_path = Buildr::Util.win_os? && path =~ /^\/[a-zA-Z]:\// ? path[1..-1] : path |
540 URI.unescape(real_path) |
541 end |
542 |
543 protected |
544 |
545 def write_internal(options, &block) #:nodoc: |
546 raise ArgumentError, 'Either you\'re attempting to write a file to another host (which we don\'t support), or you used two slashes by mistake, where you should have file:///<path>.' if host |
547 temp = Tempfile.new(File.basename(path)) |
548 temp.binmode |
549 with_progress_bar options[:progress] && options[:size], path.split('/').last, options[:size] || 0 do |progress| |
550 while chunk = yield(RW_CHUNK_SIZE) |
551 temp.write chunk |
552 progress << chunk |
553 end |
554 end |
555 temp.close |
556 mkpath File.dirname(real_path) |
557 mv temp.path, real_path |
558 real_path |
559 end |
560 |
561 @@schemes['FILE'] = FILE |
562 |
563 end |
564 |
565 end |
Generated on 2011-07-06 23:35:37 -0700 with rcov 0.9.8