PoshCode Logo PowerShell Code Repository

Get-WebFile 4.1 by Joel Bennett 30 months ago (modification of post by Joel Bennett view diff)
View followups from Joel Bennett | diff | embed code: <script type="text/javascript" src="http://PoshCode.org/embed/3275"></script>download | new post

PREVIEWUNFINISHED WORK ;-)

This is a big rewrite of Get-WebFile as a Module (I call it WebGet.psm1) to add a lot of the features of Invoke-WebRequest in PowerShell 3 (like sessions, passing body & headers, supporting POST, PUT, etc). It’s not done, but it should already be better than it was (I haven’t tested it a lot, honestly). Try it and see if you like it, and tell me how it’s broken or what it’s missing.

  1. function ConvertTo-Dictionary {
  2.     param(
  3.         [Parameter(Mandatory=$true,ValueFromPipeline=$true,ParameterSetName="Hashtable")]
  4.         [Hashtable[]]$Hashtable,
  5.  
  6.         [Parameter(Mandatory=$true,ValueFromPipelineByPropertyName=$true,ParameterSetName="WebHeaders")]
  7.         [System.Collections.Specialized.NameObjectCollectionBase]$Headers,
  8.  
  9.         [Parameter(Mandatory=$true,ParameterSetName="Hashtable")]
  10.         [Type]$TKey,
  11.  
  12.         [Parameter(Mandatory=$true,ParameterSetName="Hashtable")]
  13.         [Type]$Tvalue
  14.     )
  15.     begin {
  16.         switch($PSCmdlet.ParameterSetName) {
  17.             "Hashtable" {
  18.                 $dictionary = New-Object "System.Collections.Generic.Dictionary[[$($TKey.FullName)],[$($TValue.FullName)]]"
  19.             }
  20.             "WebHeaders" {
  21.                 $dictionary = New-Object "System.Collections.Generic.Dictionary[[String],[String]]"
  22.             }
  23.         }
  24.     }
  25.     process {
  26.         switch($PSCmdlet.ParameterSetName) {
  27.             "Hashtable" {
  28.                 foreach($ht in $Hashtable) {
  29.                     foreach($key in $ht.Keys) {
  30.                         $dictionary.Add( $key, $ht.$key )
  31.                     }
  32.                 }
  33.             }
  34.             "WebHeaders" {
  35.                 foreach($key in $Headers.AllKeys) {
  36.                     $dictionary.Add($key, $Headers[$key])
  37.                 }
  38.             }
  39.         }
  40.     }
  41.     end { return $dictionary }
  42. }
  43. function ConvertFrom-Dictionary {
  44.     [CmdletBinding()]
  45.     param($Dictionary, [Switch]$Encode)
  46.     foreach($key in $Dictionary.Keys) {
  47.         "{0} = {1}" -f $key, $( if($Encode) { [System.Net.WebUtility]::UrlEncode( $Dictionary.$key ) } else { $Dictionary.$key } )
  48.     }
  49. }
  50.  
  51. ## Get-WebFile (aka wget for PowerShell)
  52. function Invoke-Web {
  53. #.Synopsis
  54. #  Downloads a file or page from the web, or sends web API posts/requests
  55. #.Description
  56. #  Creates an HttpWebRequest to download a web file or post data
  57. #.Example
  58. #  Invoke-Web http://PoshCode.org/PoshCode.psm1
  59. #
  60. #  Downloads the latest version of the PoshCode module to the current directory
  61. #.Example
  62. #  Invoke-Web http://PoshCode.org/PoshCode.psm1 ~\Documents\WindowsPowerShell\Modules\PoshCode\
  63. #
  64. #  Downloads the latest version of the PoshCode module to the default PoshCode module directory...
  65. #.Example
  66. #  $RssItems = @(([xml](Invoke-Web http://poshcode.org/api/ -passthru)).rss.channel.GetElementsByTagName("item"))
  67. #
  68. #  Returns the most recent items from the PoshCode.org RSS feed
  69. #.Notes
  70. #  History:
  71. #  v4.1  - Reworked most of it with PowerShell 3's Invoke-WebRequest as inspiration
  72. #        - Added a bunch of parameters, the ability to do PUTs etc., and session/cookie persistence
  73. #        - Did NOT parse the return code and get you the FORMs the way PowerShell 3 does -- upgrade! ;)
  74. #  v3.12 - Added full help
  75. #  v3.9 - Fixed and replaced the Set-DownloadFlag
  76. #  v3.7 - Removed the Set-DownloadFlag code because it was throwing on Windows 7:
  77. #         "Attempted to read or write protected memory."
  78. #  v3.6.6 Add UserAgent calculation and parameter
  79. #  v3.6.5 Add file-name guessing and cleanup
  80. #  v3.6 - Add -Passthru switch to output TEXT files
  81. #  v3.5 - Add -Quiet switch to turn off the progress reports ...
  82. #  v3.4 - Add progress report for files which don't report size
  83. #  v3.3 - Add progress report for files which report their size
  84. #  v3.2 - Use the pure Stream object because StreamWriter is based on TextWriter:
  85. #         it was messing up binary files, and making mistakes with extended characters in text
  86. #  v3.1 - Unwrap the filename when it has quotes around it
  87. #  v3   - rewritten completely using HttpWebRequest + HttpWebResponse to figure out the file name, if possible
  88. #  v2   - adds a ton of parsing to make the output pretty
  89. #         added measuring the scripts involved in the command, (uses Tokenizer)
  90. [CmdletBinding(DefaultParameterSetName="NoSession")]
  91.    param(
  92.       #  The URL of the file/page to download
  93.       [Parameter(Mandatory=$true,Position=0)]
  94.       [System.Uri][Alias("Url")]$Uri # = (Read-Host "The URL to download")
  95.    ,
  96.       #  Specifies the body of the request. The body is the content of the request that follows the headers.
  97.       #  You can also pipe a request body to Invoke-WebRequest
  98.       #  Note that you should probably set the ContentType if you're setting the Body
  99.       [Parameter(ValueFromPipeline=$true)]
  100.       $Body
  101.    ,
  102.       # Specifies the content type of the web request, such as "application/x-www-form-urlencoded" (defaults to "application/x-www-form-urlencoded" if the Body is set to a hashtable, dictionary, or other NameValueCollection)
  103.       [String]$ContentType
  104.    ,
  105.       #  Specifies the client certificate that is used for a secure web request. Enter a variable that contains a certificate or a command or expression that gets the certificate.
  106.       #  To find a certificate, use Get-PfxCertificate or use the Get-ChildItem cmdlet in the Certificate (Cert:) drive. If the certificate is not valid or does not have sufficient authority, the command fails.
  107.       [System.Security.Cryptography.X509Certificates.X509Certificate[]]
  108.       $Certificate
  109.    ,
  110.       #  Sends the results to the specified output file. Enter a path and file name. If you omit the path, the default is the current location.
  111.       #  By default, Invoke-WebRequest returns the results to the pipeline. To send the results to a file and to the pipeline, use the Passthru parameter.
  112.       [string]$OutFile
  113.    ,
  114.       #  Leave the file unblocked instead of blocked
  115.       [Switch]$Unblocked
  116.    ,
  117.       #  Rather than saving the downloaded content to a file, output it.  
  118.       #  This is for text documents like web pages and rss feeds, and allows you to avoid temporarily caching the text in a file.
  119.       [switch]$Passthru
  120.    ,
  121.       #  Supresses the Write-Progress during download
  122.       [switch]$Quiet
  123.    ,
  124.       # Specifies a name for the session variable. Enter a variable name without the dollar sign ($) symbol.
  125.       # When you use the session variable in a web request, the variable is populated with a WebRequestSession object.
  126.       # You cannot use the SessionVariable and WebSession parameters in the same command
  127.       [Parameter(Mandatory=$true,ParameterSetName="CreateSession")]
  128.       [String]$SessionVariable
  129.    ,
  130.       # Specifies a web request session to store data for subsequent requests.
  131.       # You cannot use the SessionVariable and WebSession parameters in the same command
  132.       [Parameter(Mandatory=$true,ParameterSetName="UseSession")]
  133.       $WebSession
  134.    ,
  135.       #  Pass the default credentials
  136.       [switch]$UseDefaultCredentials
  137.    ,
  138.       #  Specifies a user account that has permission to send the request. The default is the current user.
  139.       #  Type a user name, such as "User01" or "Domain01\User01", or enter a PSCredential object, such as one generated by the Get-Credential cmdlet.
  140.       [System.Management.Automation.PSCredential]
  141.       [System.Management.Automation.Credential()]
  142.       [Alias("")]$Credential = [System.Management.Automation.PSCredential]::Empty
  143.    ,
  144.       # Sets the KeepAlive value in the HTTP header to False. By default, KeepAlive is True. KeepAlive establishes a persistent connection to the server to facilitate subsequent requests.
  145.       $DisableKeepAlive
  146.    ,
  147.       # Specifies the headers for the web request. Enter a hash table or dictionary.
  148.       [System.Collections.IDictionary]$Headers
  149.    ,
  150.       # Determines how many times Windows PowerShell redirects a connection to an alternate Uniform Resource Identifier (URI) before the connection fails.
  151.       # Our default value is 5 (but .Net's default is 50). A value of 0 (zero) prevents all redirection.
  152.       [int]$MaximumRedirection = 5
  153.    ,
  154.       # Specifies the method used for the web request. Valid values are Default, Delete, Get, Head, Options, Post, Put, and Trace. Default value is Get.
  155.       [ValidateSet("Default", "Delete", "Get", "Head", "Options", "Post", "Put", "Trace")]
  156.       [String]$Method = "Get"
  157.    ,
  158.       # Uses a proxy server for the request, rather than connecting directly to the Internet resource. Enter the URI of a network proxy server.
  159.       # Note: if you have a default proxy configured in your internet settings, there is no need to set it here.
  160.       [Uri]$Proxy
  161.     ,
  162.       #  Pass the default credentials to the Proxy
  163.       [switch]$ProxyUseDefaultCredentials
  164.    ,
  165.       #  Pass specific credentials to the Proxy
  166.       [System.Management.Automation.PSCredential]
  167.       [System.Management.Automation.Credential()]
  168.       $ProxyCredential= [System.Management.Automation.PSCredential]::Empty
  169.    ,
  170.       #  Text to include at the front of the UserAgent string
  171.       [string]$UserAgent = "Mozilla/5.0 (Windows NT; Windows NT $([Environment]::OSVersion.Version.ToString(2)); $PSUICulture) WindowsPowerShell/$($PSVersionTable.PSVersion.ToString(2)); PoshCode/4.0; http://PoshCode.org"    
  172.    )
  173.  
  174. process {
  175.    Write-Verbose "Downloading '$Uri'"
  176.    $EAP,$ErrorActionPreference = $ErrorActionPreference, "Stop"
  177.    $request = [System.Net.HttpWebRequest]::Create($Uri)
  178.    if($DebugPreference -ne "SilentlyContinue") {
  179.       Set-Variable WebRequest -Scope 2 -Value $request
  180.    }
  181.  
  182.    $ErrorActionPreference = $EAP
  183.    # Not everything is a GET request ...
  184.    $request.Method = $Method.ToUpper()
  185.  
  186.    # Now that we have a web request, we'll use the session values first if we have any
  187.    if($WebSession) {
  188.       $request.CookieContainer = $WebSession.Cookies
  189.       $request.Headers = $WebSession.Headers
  190.       if($WebSession.UseDefaultCredentials) {
  191.          $request.UseDefaultCredentials
  192.       } elseif($WebSession.Credentials) {
  193.          $request.Credentials = $WebSession.Credentials
  194.       }
  195.       $request.ClientCertificates = $WebSession.Certificates
  196.       $request.UserAgent = $WebSession.UserAgent
  197.       $request.Proxy = $WebSession.Proxy
  198.       $request.MaximumAutomaticRedirections = $WebSession.MaximumRedirection
  199.    } else {
  200.        $request.CookieContainer = $Cookies = New-Object System.Net.CookieContainer
  201.    }
  202.    
  203.    # And override session values with user values if they provided any
  204.    $request.UserAgent = $UserAgent
  205.    $request.MaximumAutomaticRedirections = $MaximumRedirection
  206.    $request.KeepAlive = !$DisableKeepAlive
  207.  
  208.  
  209.    # Authentication normally uses EITHER credentials or certificates, but what do I know ...
  210.    if($Certificate) {
  211.       $request.ClientCertificates.AddRange($Certificate)
  212.    }
  213.    if($UseDefaultCredentials) {
  214.       $request.UseDefaultCredentials = $true
  215.    } elseif($Credential -ne [System.Management.Automation.PSCredential]::Empty) {
  216.       $request.Credentials = $Credential.GetNetworkCredential()
  217.    }
  218.  
  219.    # You don't have to specify a proxy to specify proxy credentials (maybe your default proxy takes creds)
  220.    if($Proxy) { $request.Proxy = New-Object System.Net.WebProxy $Proxy }
  221.    if($request.Proxy -ne $null) {
  222.       if($ProxyUseDefaultCredentials) {
  223.          $request.Proxy.Credentials = [System.Net.CredentialCache]::DefaultNetworkCredentials
  224.       } elseif($ProxyCredentials -ne [System.Management.Automation.PSCredential]::Empty) {
  225.          $request.Proxy.Credentials = $ProxyCredentials
  226.       }
  227.    }
  228.  
  229.    if($SessionVariable) {
  230.       Set-Variable $SessionVariable -Scope 1 -Value $WebSession
  231.    }
  232.    
  233.    if($Headers) {
  234.       foreach($h in $Headers.Keys) {
  235.          $request.Headers.Add($h, $Headers[$h])
  236.       }
  237.    }
  238.  
  239.    if($Body) {
  240.       $writer = $request.GetRequestStream();
  241.       if($Body -is [System.Collections.IDictionary] -or $Body -is [System.Collections.Specialized.NameObjectCollectionBase]) {
  242.          if(!$ContentType) {
  243.             $ContentType = "application/x-www-form-urlencoded"
  244.          }
  245.          [String]$Body = ConvertFrom-Dictionary $Body -Encode $($ContentType -eq "application/x-www-form-urlencoded")
  246.       } else {
  247.          $Body = $Body | Out-String
  248.       }
  249.  
  250.       $encoding = New-Object System.Text.ASCIIEncoding
  251.       $bytes = $encoding.GetBytes($Body);
  252.       $request.ContentType = $ContentType
  253.       $request.ContentLength = $bytes.Length
  254.       $writer.Write($bytes, 0, $bytes.Length)
  255.       $writer.Close()
  256.    }
  257.  
  258.    try {
  259.       $response = $request.GetResponse();
  260.        if($DebugPreference -ne "SilentlyContinue") {
  261.           Set-Variable WebResponse -Scope 2 -Value $response
  262.        }
  263.    } catch [System.Net.WebException] {
  264.       Write-Error $_.Exception -Category ResourceUnavailable
  265.       return
  266.    } catch { # Extra catch just in case, I can't remember what might fall here
  267.       Write-Error $_.Exception -Category NotImplemented
  268.       return
  269.    }
  270.  
  271.    Write-Verbose "Retrieved $($Response.ResponseUri)"
  272.    if((Test-Path variable:response) -and $response.StatusCode -eq 200) {
  273.       # Magics to figure out a file location based on the response
  274.       if($OutFile -and !(Split-Path $OutFile)) {
  275.          $OutFile = Join-Path (Convert-Path (Get-Location -PSProvider "FileSystem")) $OutFile
  276.       }
  277.       elseif((!$Passthru -and !$OutFile) -or ($OutFile -and (Test-Path -PathType "Container" $OutFile)))
  278.       {
  279.          [string]$OutFile = ([regex]'(?i)filename=(.*)$').Match( $response.Headers["Content-Disposition"] ).Groups[1].Value
  280.          $OutFile = $OutFile.trim("\/""'")
  281.          
  282.          $ofs = ""
  283.          $OutFile = [Regex]::Replace($OutFile, "[$([Regex]::Escape(""$([System.IO.Path]::GetInvalidPathChars())$([IO.Path]::AltDirectorySeparatorChar)$([IO.Path]::DirectorySeparatorChar)""))]", "_")
  284.          $ofs = " "
  285.          
  286.          if(!$OutFile) {
  287.             $OutFile = $response.ResponseUri.Segments[-1]
  288.             $OutFile = $OutFile.trim("\/")
  289.             if(!$OutFile) {
  290.                $OutFile = Read-Host "Please provide a file name"
  291.             }
  292.             $OutFile = $OutFile.trim("\/")
  293.             if(!([IO.FileInfo]$OutFile).Extension) {
  294.                $OutFile = $OutFile + "." + $response.ContentType.Split(";")[0].Split("/")[1]
  295.             }
  296.          }
  297.          $OutFile = Join-Path (Convert-Path (Get-Location -PSProvider "FileSystem")) $OutFile
  298.       }
  299.  
  300.       if($Passthru) {
  301.          $encoding = [System.Text.Encoding]::GetEncoding( $response.CharacterSet )
  302.          [string]$output = ""
  303.       }
  304.  
  305.       [int]$goal = $response.ContentLength
  306.       $reader = $response.GetResponseStream()
  307.       if($OutFile) {
  308.          try {
  309.             $writer = new-object System.IO.FileStream $OutFile, "Create"
  310.          } catch { # Catch just in case, lots of things could go wrong ...
  311.             Write-Error $_.Exception -Category WriteError
  312.             return
  313.          }
  314.       }
  315.  
  316.       [byte[]]$buffer = new-object byte[] 4096
  317.       [int]$total = [int]$count = 0
  318.       do
  319.       {
  320.          $count = $reader.Read($buffer, 0, $buffer.Length);
  321.          Write-Verbose "Read $count"
  322.          if($OutFile) {
  323.             $writer.Write($buffer, 0, $count);
  324.          }
  325.          if($Passthru){
  326.             $output += $encoding.GetString($buffer,0,$count)
  327.          } elseif(!$quiet) {
  328.             $total += $count
  329.             if($goal -gt 0) {
  330.                Write-Progress "Downloading $Uri" "Saving $total of $goal" -id 0 -percentComplete (($total/$goal)*100)
  331.             } else {
  332.                Write-Progress "Downloading $Uri" "Saving $total bytes..." -id 0
  333.             }
  334.          }
  335.       } while ($count -gt 0)
  336.      
  337.       $reader.Close()
  338.       if($OutFile) {
  339.          $writer.Flush()
  340.          $writer.Close()
  341.       }
  342.       if($Passthru){
  343.          $output
  344.       }
  345.    }
  346.    if(Test-Path variable:response) { $response.Close(); }
  347.  
  348.    if($SessionVariable) {
  349.           Set-Variable $SessionVariable -Scope 1 -Value ([PSCustomObject]@{
  350.           Headers               = ConvertTo-Dictionary -Headers $request.Headers
  351.           Cookies               = $response.Cookies
  352.           UseDefaultCredentials = $request.UseDefaultCredentials
  353.           Credentials           = $request.Credentials
  354.           Certificates          = $request.ClientCertificates
  355.           UserAgent             = $request.UserAgent
  356.           Proxy                 = $request.Proxy
  357.           MaximumRedirection    = $request.MaximumAutomaticRedirections
  358.        })
  359.     }
  360.     if($WebSession) {
  361.        $WebSession.Cookies      = $response.Cookies
  362.     }
  363. }}

Submit a correction or amendment below (
click here to make a fresh posting)
After submitting an amendment, you'll be able to view the differences between the old and new posts easily.

Syntax highlighting:


Remember me