client.go 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989
  1. // Package storage provides clients for Microsoft Azure Storage Services.
  2. package storage
  3. // Copyright 2017 Microsoft Corporation
  4. //
  5. // Licensed under the Apache License, Version 2.0 (the "License");
  6. // you may not use this file except in compliance with the License.
  7. // You may obtain a copy of the License at
  8. //
  9. // http://www.apache.org/licenses/LICENSE-2.0
  10. //
  11. // Unless required by applicable law or agreed to in writing, software
  12. // distributed under the License is distributed on an "AS IS" BASIS,
  13. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. // See the License for the specific language governing permissions and
  15. // limitations under the License.
  16. import (
  17. "bufio"
  18. "encoding/base64"
  19. "encoding/json"
  20. "encoding/xml"
  21. "errors"
  22. "fmt"
  23. "io"
  24. "io/ioutil"
  25. "mime"
  26. "mime/multipart"
  27. "net/http"
  28. "net/url"
  29. "regexp"
  30. "runtime"
  31. "strconv"
  32. "strings"
  33. "time"
  34. "github.com/Azure/azure-sdk-for-go/version"
  35. "github.com/Azure/go-autorest/autorest"
  36. "github.com/Azure/go-autorest/autorest/azure"
  37. )
  38. const (
  39. // DefaultBaseURL is the domain name used for storage requests in the
  40. // public cloud when a default client is created.
  41. DefaultBaseURL = "core.windows.net"
  42. // DefaultAPIVersion is the Azure Storage API version string used when a
  43. // basic client is created.
  44. DefaultAPIVersion = "2016-05-31"
  45. defaultUseHTTPS = true
  46. defaultRetryAttempts = 5
  47. defaultRetryDuration = time.Second * 5
  48. // StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator
  49. StorageEmulatorAccountName = "devstoreaccount1"
  50. // StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator
  51. StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
  52. blobServiceName = "blob"
  53. tableServiceName = "table"
  54. queueServiceName = "queue"
  55. fileServiceName = "file"
  56. storageEmulatorBlob = "127.0.0.1:10000"
  57. storageEmulatorTable = "127.0.0.1:10002"
  58. storageEmulatorQueue = "127.0.0.1:10001"
  59. userAgentHeader = "User-Agent"
  60. userDefinedMetadataHeaderPrefix = "x-ms-meta-"
  61. connectionStringAccountName = "accountname"
  62. connectionStringAccountKey = "accountkey"
  63. connectionStringEndpointSuffix = "endpointsuffix"
  64. connectionStringEndpointProtocol = "defaultendpointsprotocol"
  65. connectionStringBlobEndpoint = "blobendpoint"
  66. connectionStringFileEndpoint = "fileendpoint"
  67. connectionStringQueueEndpoint = "queueendpoint"
  68. connectionStringTableEndpoint = "tableendpoint"
  69. connectionStringSAS = "sharedaccesssignature"
  70. )
  71. var (
  72. validStorageAccount = regexp.MustCompile("^[0-9a-z]{3,24}$")
  73. defaultValidStatusCodes = []int{
  74. http.StatusRequestTimeout, // 408
  75. http.StatusInternalServerError, // 500
  76. http.StatusBadGateway, // 502
  77. http.StatusServiceUnavailable, // 503
  78. http.StatusGatewayTimeout, // 504
  79. }
  80. )
  81. // Sender sends a request
  82. type Sender interface {
  83. Send(*Client, *http.Request) (*http.Response, error)
  84. }
  85. // DefaultSender is the default sender for the client. It implements
  86. // an automatic retry strategy.
  87. type DefaultSender struct {
  88. RetryAttempts int
  89. RetryDuration time.Duration
  90. ValidStatusCodes []int
  91. attempts int // used for testing
  92. }
  93. // Send is the default retry strategy in the client
  94. func (ds *DefaultSender) Send(c *Client, req *http.Request) (resp *http.Response, err error) {
  95. rr := autorest.NewRetriableRequest(req)
  96. for attempts := 0; attempts < ds.RetryAttempts; attempts++ {
  97. err = rr.Prepare()
  98. if err != nil {
  99. return resp, err
  100. }
  101. resp, err = c.HTTPClient.Do(rr.Request())
  102. if err != nil || !autorest.ResponseHasStatusCode(resp, ds.ValidStatusCodes...) {
  103. return resp, err
  104. }
  105. drainRespBody(resp)
  106. autorest.DelayForBackoff(ds.RetryDuration, attempts, req.Cancel)
  107. ds.attempts = attempts
  108. }
  109. ds.attempts++
  110. return resp, err
  111. }
  112. // Client is the object that needs to be constructed to perform
  113. // operations on the storage account.
  114. type Client struct {
  115. // HTTPClient is the http.Client used to initiate API
  116. // requests. http.DefaultClient is used when creating a
  117. // client.
  118. HTTPClient *http.Client
  119. // Sender is an interface that sends the request. Clients are
  120. // created with a DefaultSender. The DefaultSender has an
  121. // automatic retry strategy built in. The Sender can be customized.
  122. Sender Sender
  123. accountName string
  124. accountKey []byte
  125. useHTTPS bool
  126. UseSharedKeyLite bool
  127. baseURL string
  128. apiVersion string
  129. userAgent string
  130. sasClient bool
  131. accountSASToken url.Values
  132. }
  133. type odataResponse struct {
  134. resp *http.Response
  135. odata odataErrorWrapper
  136. }
  137. // AzureStorageServiceError contains fields of the error response from
  138. // Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx
  139. // Some fields might be specific to certain calls.
  140. type AzureStorageServiceError struct {
  141. Code string `xml:"Code"`
  142. Message string `xml:"Message"`
  143. AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
  144. QueryParameterName string `xml:"QueryParameterName"`
  145. QueryParameterValue string `xml:"QueryParameterValue"`
  146. Reason string `xml:"Reason"`
  147. Lang string
  148. StatusCode int
  149. RequestID string
  150. Date string
  151. APIVersion string
  152. }
  153. type odataErrorMessage struct {
  154. Lang string `json:"lang"`
  155. Value string `json:"value"`
  156. }
  157. type odataError struct {
  158. Code string `json:"code"`
  159. Message odataErrorMessage `json:"message"`
  160. }
  161. type odataErrorWrapper struct {
  162. Err odataError `json:"odata.error"`
  163. }
  164. // UnexpectedStatusCodeError is returned when a storage service responds with neither an error
  165. // nor with an HTTP status code indicating success.
  166. type UnexpectedStatusCodeError struct {
  167. allowed []int
  168. got int
  169. inner error
  170. }
  171. func (e UnexpectedStatusCodeError) Error() string {
  172. s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) }
  173. got := s(e.got)
  174. expected := []string{}
  175. for _, v := range e.allowed {
  176. expected = append(expected, s(v))
  177. }
  178. return fmt.Sprintf("storage: status code from service response is %s; was expecting %s. Inner error: %+v", got, strings.Join(expected, " or "), e.inner)
  179. }
  180. // Got is the actual status code returned by Azure.
  181. func (e UnexpectedStatusCodeError) Got() int {
  182. return e.got
  183. }
  184. // Inner returns any inner error info.
  185. func (e UnexpectedStatusCodeError) Inner() error {
  186. return e.inner
  187. }
  188. // NewClientFromConnectionString creates a Client from the connection string.
  189. func NewClientFromConnectionString(input string) (Client, error) {
  190. // build a map of connection string key/value pairs
  191. parts := map[string]string{}
  192. for _, pair := range strings.Split(input, ";") {
  193. if pair == "" {
  194. continue
  195. }
  196. equalDex := strings.IndexByte(pair, '=')
  197. if equalDex <= 0 {
  198. return Client{}, fmt.Errorf("Invalid connection segment %q", pair)
  199. }
  200. value := strings.TrimSpace(pair[equalDex+1:])
  201. key := strings.TrimSpace(strings.ToLower(pair[:equalDex]))
  202. parts[key] = value
  203. }
  204. // TODO: validate parameter sets?
  205. if parts[connectionStringAccountName] == StorageEmulatorAccountName {
  206. return NewEmulatorClient()
  207. }
  208. if parts[connectionStringSAS] != "" {
  209. endpoint := ""
  210. if parts[connectionStringBlobEndpoint] != "" {
  211. endpoint = parts[connectionStringBlobEndpoint]
  212. } else if parts[connectionStringFileEndpoint] != "" {
  213. endpoint = parts[connectionStringFileEndpoint]
  214. } else if parts[connectionStringQueueEndpoint] != "" {
  215. endpoint = parts[connectionStringQueueEndpoint]
  216. } else {
  217. endpoint = parts[connectionStringTableEndpoint]
  218. }
  219. return NewAccountSASClientFromEndpointToken(endpoint, parts[connectionStringSAS])
  220. }
  221. useHTTPS := defaultUseHTTPS
  222. if parts[connectionStringEndpointProtocol] != "" {
  223. useHTTPS = parts[connectionStringEndpointProtocol] == "https"
  224. }
  225. return NewClient(parts[connectionStringAccountName], parts[connectionStringAccountKey],
  226. parts[connectionStringEndpointSuffix], DefaultAPIVersion, useHTTPS)
  227. }
  228. // NewBasicClient constructs a Client with given storage service name and
  229. // key.
  230. func NewBasicClient(accountName, accountKey string) (Client, error) {
  231. if accountName == StorageEmulatorAccountName {
  232. return NewEmulatorClient()
  233. }
  234. return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS)
  235. }
  236. // NewBasicClientOnSovereignCloud constructs a Client with given storage service name and
  237. // key in the referenced cloud.
  238. func NewBasicClientOnSovereignCloud(accountName, accountKey string, env azure.Environment) (Client, error) {
  239. if accountName == StorageEmulatorAccountName {
  240. return NewEmulatorClient()
  241. }
  242. return NewClient(accountName, accountKey, env.StorageEndpointSuffix, DefaultAPIVersion, defaultUseHTTPS)
  243. }
  244. //NewEmulatorClient contructs a Client intended to only work with Azure
  245. //Storage Emulator
  246. func NewEmulatorClient() (Client, error) {
  247. return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false)
  248. }
  249. // NewClient constructs a Client. This should be used if the caller wants
  250. // to specify whether to use HTTPS, a specific REST API version or a custom
  251. // storage endpoint than Azure Public Cloud.
  252. func NewClient(accountName, accountKey, serviceBaseURL, apiVersion string, useHTTPS bool) (Client, error) {
  253. var c Client
  254. if !IsValidStorageAccount(accountName) {
  255. return c, fmt.Errorf("azure: account name is not valid: it must be between 3 and 24 characters, and only may contain numbers and lowercase letters: %v", accountName)
  256. } else if accountKey == "" {
  257. return c, fmt.Errorf("azure: account key required")
  258. } else if serviceBaseURL == "" {
  259. return c, fmt.Errorf("azure: base storage service url required")
  260. }
  261. key, err := base64.StdEncoding.DecodeString(accountKey)
  262. if err != nil {
  263. return c, fmt.Errorf("azure: malformed storage account key: %v", err)
  264. }
  265. c = Client{
  266. HTTPClient: http.DefaultClient,
  267. accountName: accountName,
  268. accountKey: key,
  269. useHTTPS: useHTTPS,
  270. baseURL: serviceBaseURL,
  271. apiVersion: apiVersion,
  272. sasClient: false,
  273. UseSharedKeyLite: false,
  274. Sender: &DefaultSender{
  275. RetryAttempts: defaultRetryAttempts,
  276. ValidStatusCodes: defaultValidStatusCodes,
  277. RetryDuration: defaultRetryDuration,
  278. },
  279. }
  280. c.userAgent = c.getDefaultUserAgent()
  281. return c, nil
  282. }
  283. // IsValidStorageAccount checks if the storage account name is valid.
  284. // See https://docs.microsoft.com/en-us/azure/storage/storage-create-storage-account
  285. func IsValidStorageAccount(account string) bool {
  286. return validStorageAccount.MatchString(account)
  287. }
  288. // NewAccountSASClient contructs a client that uses accountSAS authorization
  289. // for its operations.
  290. func NewAccountSASClient(account string, token url.Values, env azure.Environment) Client {
  291. return newSASClient(account, env.StorageEndpointSuffix, token)
  292. }
  293. // NewAccountSASClientFromEndpointToken constructs a client that uses accountSAS authorization
  294. // for its operations using the specified endpoint and SAS token.
  295. func NewAccountSASClientFromEndpointToken(endpoint string, sasToken string) (Client, error) {
  296. u, err := url.Parse(endpoint)
  297. if err != nil {
  298. return Client{}, err
  299. }
  300. _, err = url.ParseQuery(sasToken)
  301. if err != nil {
  302. return Client{}, err
  303. }
  304. u.RawQuery = sasToken
  305. return newSASClientFromURL(u)
  306. }
  307. func newSASClient(accountName, baseURL string, sasToken url.Values) Client {
  308. c := Client{
  309. HTTPClient: http.DefaultClient,
  310. apiVersion: DefaultAPIVersion,
  311. sasClient: true,
  312. Sender: &DefaultSender{
  313. RetryAttempts: defaultRetryAttempts,
  314. ValidStatusCodes: defaultValidStatusCodes,
  315. RetryDuration: defaultRetryDuration,
  316. },
  317. accountName: accountName,
  318. baseURL: baseURL,
  319. accountSASToken: sasToken,
  320. }
  321. c.userAgent = c.getDefaultUserAgent()
  322. // Get API version and protocol from token
  323. c.apiVersion = sasToken.Get("sv")
  324. c.useHTTPS = sasToken.Get("spr") == "https"
  325. return c
  326. }
  327. func newSASClientFromURL(u *url.URL) (Client, error) {
  328. // the host name will look something like this
  329. // - foo.blob.core.windows.net
  330. // "foo" is the account name
  331. // "core.windows.net" is the baseURL
  332. // find the first dot to get account name
  333. i1 := strings.IndexByte(u.Host, '.')
  334. if i1 < 0 {
  335. return Client{}, fmt.Errorf("failed to find '.' in %s", u.Host)
  336. }
  337. // now find the second dot to get the base URL
  338. i2 := strings.IndexByte(u.Host[i1+1:], '.')
  339. if i2 < 0 {
  340. return Client{}, fmt.Errorf("failed to find '.' in %s", u.Host[i1+1:])
  341. }
  342. sasToken := u.Query()
  343. c := newSASClient(u.Host[:i1], u.Host[i1+i2+2:], sasToken)
  344. if spr := sasToken.Get("spr"); spr == "" {
  345. // infer from URL if not in the query params set
  346. c.useHTTPS = u.Scheme == "https"
  347. }
  348. return c, nil
  349. }
  350. func (c Client) isServiceSASClient() bool {
  351. return c.sasClient && c.accountSASToken == nil
  352. }
  353. func (c Client) isAccountSASClient() bool {
  354. return c.sasClient && c.accountSASToken != nil
  355. }
  356. func (c Client) getDefaultUserAgent() string {
  357. return fmt.Sprintf("Go/%s (%s-%s) azure-storage-go/%s api-version/%s",
  358. runtime.Version(),
  359. runtime.GOARCH,
  360. runtime.GOOS,
  361. version.Number,
  362. c.apiVersion,
  363. )
  364. }
  365. // AddToUserAgent adds an extension to the current user agent
  366. func (c *Client) AddToUserAgent(extension string) error {
  367. if extension != "" {
  368. c.userAgent = fmt.Sprintf("%s %s", c.userAgent, extension)
  369. return nil
  370. }
  371. return fmt.Errorf("Extension was empty, User Agent stayed as %s", c.userAgent)
  372. }
  373. // protectUserAgent is used in funcs that include extraheaders as a parameter.
  374. // It prevents the User-Agent header to be overwritten, instead if it happens to
  375. // be present, it gets added to the current User-Agent. Use it before getStandardHeaders
  376. func (c *Client) protectUserAgent(extraheaders map[string]string) map[string]string {
  377. if v, ok := extraheaders[userAgentHeader]; ok {
  378. c.AddToUserAgent(v)
  379. delete(extraheaders, userAgentHeader)
  380. }
  381. return extraheaders
  382. }
  383. func (c Client) getBaseURL(service string) *url.URL {
  384. scheme := "http"
  385. if c.useHTTPS {
  386. scheme = "https"
  387. }
  388. host := ""
  389. if c.accountName == StorageEmulatorAccountName {
  390. switch service {
  391. case blobServiceName:
  392. host = storageEmulatorBlob
  393. case tableServiceName:
  394. host = storageEmulatorTable
  395. case queueServiceName:
  396. host = storageEmulatorQueue
  397. }
  398. } else {
  399. host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL)
  400. }
  401. return &url.URL{
  402. Scheme: scheme,
  403. Host: host,
  404. }
  405. }
  406. func (c Client) getEndpoint(service, path string, params url.Values) string {
  407. u := c.getBaseURL(service)
  408. // API doesn't accept path segments not starting with '/'
  409. if !strings.HasPrefix(path, "/") {
  410. path = fmt.Sprintf("/%v", path)
  411. }
  412. if c.accountName == StorageEmulatorAccountName {
  413. path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path)
  414. }
  415. u.Path = path
  416. u.RawQuery = params.Encode()
  417. return u.String()
  418. }
  419. // AccountSASTokenOptions includes options for constructing
  420. // an account SAS token.
  421. // https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas
  422. type AccountSASTokenOptions struct {
  423. APIVersion string
  424. Services Services
  425. ResourceTypes ResourceTypes
  426. Permissions Permissions
  427. Start time.Time
  428. Expiry time.Time
  429. IP string
  430. UseHTTPS bool
  431. }
  432. // Services specify services accessible with an account SAS.
  433. type Services struct {
  434. Blob bool
  435. Queue bool
  436. Table bool
  437. File bool
  438. }
  439. // ResourceTypes specify the resources accesible with an
  440. // account SAS.
  441. type ResourceTypes struct {
  442. Service bool
  443. Container bool
  444. Object bool
  445. }
  446. // Permissions specifies permissions for an accountSAS.
  447. type Permissions struct {
  448. Read bool
  449. Write bool
  450. Delete bool
  451. List bool
  452. Add bool
  453. Create bool
  454. Update bool
  455. Process bool
  456. }
  457. // GetAccountSASToken creates an account SAS token
  458. // See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas
  459. func (c Client) GetAccountSASToken(options AccountSASTokenOptions) (url.Values, error) {
  460. if options.APIVersion == "" {
  461. options.APIVersion = c.apiVersion
  462. }
  463. if options.APIVersion < "2015-04-05" {
  464. return url.Values{}, fmt.Errorf("account SAS does not support API versions prior to 2015-04-05. API version : %s", options.APIVersion)
  465. }
  466. // build services string
  467. services := ""
  468. if options.Services.Blob {
  469. services += "b"
  470. }
  471. if options.Services.Queue {
  472. services += "q"
  473. }
  474. if options.Services.Table {
  475. services += "t"
  476. }
  477. if options.Services.File {
  478. services += "f"
  479. }
  480. // build resources string
  481. resources := ""
  482. if options.ResourceTypes.Service {
  483. resources += "s"
  484. }
  485. if options.ResourceTypes.Container {
  486. resources += "c"
  487. }
  488. if options.ResourceTypes.Object {
  489. resources += "o"
  490. }
  491. // build permissions string
  492. permissions := ""
  493. if options.Permissions.Read {
  494. permissions += "r"
  495. }
  496. if options.Permissions.Write {
  497. permissions += "w"
  498. }
  499. if options.Permissions.Delete {
  500. permissions += "d"
  501. }
  502. if options.Permissions.List {
  503. permissions += "l"
  504. }
  505. if options.Permissions.Add {
  506. permissions += "a"
  507. }
  508. if options.Permissions.Create {
  509. permissions += "c"
  510. }
  511. if options.Permissions.Update {
  512. permissions += "u"
  513. }
  514. if options.Permissions.Process {
  515. permissions += "p"
  516. }
  517. // build start time, if exists
  518. start := ""
  519. if options.Start != (time.Time{}) {
  520. start = options.Start.UTC().Format(time.RFC3339)
  521. }
  522. // build expiry time
  523. expiry := options.Expiry.UTC().Format(time.RFC3339)
  524. protocol := "https,http"
  525. if options.UseHTTPS {
  526. protocol = "https"
  527. }
  528. stringToSign := strings.Join([]string{
  529. c.accountName,
  530. permissions,
  531. services,
  532. resources,
  533. start,
  534. expiry,
  535. options.IP,
  536. protocol,
  537. options.APIVersion,
  538. "",
  539. }, "\n")
  540. signature := c.computeHmac256(stringToSign)
  541. sasParams := url.Values{
  542. "sv": {options.APIVersion},
  543. "ss": {services},
  544. "srt": {resources},
  545. "sp": {permissions},
  546. "se": {expiry},
  547. "spr": {protocol},
  548. "sig": {signature},
  549. }
  550. if start != "" {
  551. sasParams.Add("st", start)
  552. }
  553. if options.IP != "" {
  554. sasParams.Add("sip", options.IP)
  555. }
  556. return sasParams, nil
  557. }
  558. // GetBlobService returns a BlobStorageClient which can operate on the blob
  559. // service of the storage account.
  560. func (c Client) GetBlobService() BlobStorageClient {
  561. b := BlobStorageClient{
  562. client: c,
  563. }
  564. b.client.AddToUserAgent(blobServiceName)
  565. b.auth = sharedKey
  566. if c.UseSharedKeyLite {
  567. b.auth = sharedKeyLite
  568. }
  569. return b
  570. }
  571. // GetQueueService returns a QueueServiceClient which can operate on the queue
  572. // service of the storage account.
  573. func (c Client) GetQueueService() QueueServiceClient {
  574. q := QueueServiceClient{
  575. client: c,
  576. }
  577. q.client.AddToUserAgent(queueServiceName)
  578. q.auth = sharedKey
  579. if c.UseSharedKeyLite {
  580. q.auth = sharedKeyLite
  581. }
  582. return q
  583. }
  584. // GetTableService returns a TableServiceClient which can operate on the table
  585. // service of the storage account.
  586. func (c Client) GetTableService() TableServiceClient {
  587. t := TableServiceClient{
  588. client: c,
  589. }
  590. t.client.AddToUserAgent(tableServiceName)
  591. t.auth = sharedKeyForTable
  592. if c.UseSharedKeyLite {
  593. t.auth = sharedKeyLiteForTable
  594. }
  595. return t
  596. }
  597. // GetFileService returns a FileServiceClient which can operate on the file
  598. // service of the storage account.
  599. func (c Client) GetFileService() FileServiceClient {
  600. f := FileServiceClient{
  601. client: c,
  602. }
  603. f.client.AddToUserAgent(fileServiceName)
  604. f.auth = sharedKey
  605. if c.UseSharedKeyLite {
  606. f.auth = sharedKeyLite
  607. }
  608. return f
  609. }
  610. func (c Client) getStandardHeaders() map[string]string {
  611. return map[string]string{
  612. userAgentHeader: c.userAgent,
  613. "x-ms-version": c.apiVersion,
  614. "x-ms-date": currentTimeRfc1123Formatted(),
  615. }
  616. }
  617. func (c Client) exec(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*http.Response, error) {
  618. headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
  619. if err != nil {
  620. return nil, err
  621. }
  622. req, err := http.NewRequest(verb, url, body)
  623. if err != nil {
  624. return nil, errors.New("azure/storage: error creating request: " + err.Error())
  625. }
  626. // http.NewRequest() will automatically set req.ContentLength for a handful of types
  627. // otherwise we will handle here.
  628. if req.ContentLength < 1 {
  629. if clstr, ok := headers["Content-Length"]; ok {
  630. if cl, err := strconv.ParseInt(clstr, 10, 64); err == nil {
  631. req.ContentLength = cl
  632. }
  633. }
  634. }
  635. for k, v := range headers {
  636. req.Header[k] = append(req.Header[k], v) // Must bypass case munging present in `Add` by using map functions directly. See https://github.com/Azure/azure-sdk-for-go/issues/645
  637. }
  638. if c.isAccountSASClient() {
  639. // append the SAS token to the query params
  640. v := req.URL.Query()
  641. v = mergeParams(v, c.accountSASToken)
  642. req.URL.RawQuery = v.Encode()
  643. }
  644. resp, err := c.Sender.Send(&c, req)
  645. if err != nil {
  646. return nil, err
  647. }
  648. if resp.StatusCode >= 400 && resp.StatusCode <= 505 {
  649. return resp, getErrorFromResponse(resp)
  650. }
  651. return resp, nil
  652. }
  653. func (c Client) execInternalJSONCommon(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, *http.Request, *http.Response, error) {
  654. headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
  655. if err != nil {
  656. return nil, nil, nil, err
  657. }
  658. req, err := http.NewRequest(verb, url, body)
  659. for k, v := range headers {
  660. req.Header.Add(k, v)
  661. }
  662. resp, err := c.Sender.Send(&c, req)
  663. if err != nil {
  664. return nil, nil, nil, err
  665. }
  666. respToRet := &odataResponse{resp: resp}
  667. statusCode := resp.StatusCode
  668. if statusCode >= 400 && statusCode <= 505 {
  669. var respBody []byte
  670. respBody, err = readAndCloseBody(resp.Body)
  671. if err != nil {
  672. return nil, nil, nil, err
  673. }
  674. requestID, date, version := getDebugHeaders(resp.Header)
  675. if len(respBody) == 0 {
  676. // no error in response body, might happen in HEAD requests
  677. err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version)
  678. return respToRet, req, resp, err
  679. }
  680. // try unmarshal as odata.error json
  681. err = json.Unmarshal(respBody, &respToRet.odata)
  682. }
  683. return respToRet, req, resp, err
  684. }
  685. func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
  686. respToRet, _, _, err := c.execInternalJSONCommon(verb, url, headers, body, auth)
  687. return respToRet, err
  688. }
  689. func (c Client) execBatchOperationJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
  690. // execute common query, get back generated request, response etc... for more processing.
  691. respToRet, req, resp, err := c.execInternalJSONCommon(verb, url, headers, body, auth)
  692. if err != nil {
  693. return nil, err
  694. }
  695. // return the OData in the case of executing batch commands.
  696. // In this case we need to read the outer batch boundary and contents.
  697. // Then we read the changeset information within the batch
  698. var respBody []byte
  699. respBody, err = readAndCloseBody(resp.Body)
  700. if err != nil {
  701. return nil, err
  702. }
  703. // outer multipart body
  704. _, batchHeader, err := mime.ParseMediaType(resp.Header["Content-Type"][0])
  705. if err != nil {
  706. return nil, err
  707. }
  708. // batch details.
  709. batchBoundary := batchHeader["boundary"]
  710. batchPartBuf, changesetBoundary, err := genBatchReader(batchBoundary, respBody)
  711. if err != nil {
  712. return nil, err
  713. }
  714. // changeset details.
  715. err = genChangesetReader(req, respToRet, batchPartBuf, changesetBoundary)
  716. if err != nil {
  717. return nil, err
  718. }
  719. return respToRet, nil
  720. }
  721. func genChangesetReader(req *http.Request, respToRet *odataResponse, batchPartBuf io.Reader, changesetBoundary string) error {
  722. changesetMultiReader := multipart.NewReader(batchPartBuf, changesetBoundary)
  723. changesetPart, err := changesetMultiReader.NextPart()
  724. if err != nil {
  725. return err
  726. }
  727. changesetPartBufioReader := bufio.NewReader(changesetPart)
  728. changesetResp, err := http.ReadResponse(changesetPartBufioReader, req)
  729. if err != nil {
  730. return err
  731. }
  732. if changesetResp.StatusCode != http.StatusNoContent {
  733. changesetBody, err := readAndCloseBody(changesetResp.Body)
  734. err = json.Unmarshal(changesetBody, &respToRet.odata)
  735. if err != nil {
  736. return err
  737. }
  738. respToRet.resp = changesetResp
  739. }
  740. return nil
  741. }
  742. func genBatchReader(batchBoundary string, respBody []byte) (io.Reader, string, error) {
  743. respBodyString := string(respBody)
  744. respBodyReader := strings.NewReader(respBodyString)
  745. // reading batchresponse
  746. batchMultiReader := multipart.NewReader(respBodyReader, batchBoundary)
  747. batchPart, err := batchMultiReader.NextPart()
  748. if err != nil {
  749. return nil, "", err
  750. }
  751. batchPartBufioReader := bufio.NewReader(batchPart)
  752. _, changesetHeader, err := mime.ParseMediaType(batchPart.Header.Get("Content-Type"))
  753. if err != nil {
  754. return nil, "", err
  755. }
  756. changesetBoundary := changesetHeader["boundary"]
  757. return batchPartBufioReader, changesetBoundary, nil
  758. }
  759. func readAndCloseBody(body io.ReadCloser) ([]byte, error) {
  760. defer body.Close()
  761. out, err := ioutil.ReadAll(body)
  762. if err == io.EOF {
  763. err = nil
  764. }
  765. return out, err
  766. }
  767. // reads the response body then closes it
  768. func drainRespBody(resp *http.Response) {
  769. io.Copy(ioutil.Discard, resp.Body)
  770. resp.Body.Close()
  771. }
  772. func serviceErrFromXML(body []byte, storageErr *AzureStorageServiceError) error {
  773. if err := xml.Unmarshal(body, storageErr); err != nil {
  774. storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body))
  775. return err
  776. }
  777. return nil
  778. }
  779. func serviceErrFromJSON(body []byte, storageErr *AzureStorageServiceError) error {
  780. odataError := odataErrorWrapper{}
  781. if err := json.Unmarshal(body, &odataError); err != nil {
  782. storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body))
  783. return err
  784. }
  785. storageErr.Code = odataError.Err.Code
  786. storageErr.Message = odataError.Err.Message.Value
  787. storageErr.Lang = odataError.Err.Message.Lang
  788. return nil
  789. }
  790. func serviceErrFromStatusCode(code int, status string, requestID, date, version string) AzureStorageServiceError {
  791. return AzureStorageServiceError{
  792. StatusCode: code,
  793. Code: status,
  794. RequestID: requestID,
  795. Date: date,
  796. APIVersion: version,
  797. Message: "no response body was available for error status code",
  798. }
  799. }
  800. func (e AzureStorageServiceError) Error() string {
  801. return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestInitiated=%s, RequestId=%s, API Version=%s, QueryParameterName=%s, QueryParameterValue=%s",
  802. e.StatusCode, e.Code, e.Message, e.Date, e.RequestID, e.APIVersion, e.QueryParameterName, e.QueryParameterValue)
  803. }
  804. // checkRespCode returns UnexpectedStatusError if the given response code is not
  805. // one of the allowed status codes; otherwise nil.
  806. func checkRespCode(resp *http.Response, allowed []int) error {
  807. for _, v := range allowed {
  808. if resp.StatusCode == v {
  809. return nil
  810. }
  811. }
  812. err := getErrorFromResponse(resp)
  813. return UnexpectedStatusCodeError{
  814. allowed: allowed,
  815. got: resp.StatusCode,
  816. inner: err,
  817. }
  818. }
  819. func (c Client) addMetadataToHeaders(h map[string]string, metadata map[string]string) map[string]string {
  820. metadata = c.protectUserAgent(metadata)
  821. for k, v := range metadata {
  822. h[userDefinedMetadataHeaderPrefix+k] = v
  823. }
  824. return h
  825. }
  826. func getDebugHeaders(h http.Header) (requestID, date, version string) {
  827. requestID = h.Get("x-ms-request-id")
  828. version = h.Get("x-ms-version")
  829. date = h.Get("Date")
  830. return
  831. }
  832. func getErrorFromResponse(resp *http.Response) error {
  833. respBody, err := readAndCloseBody(resp.Body)
  834. if err != nil {
  835. return err
  836. }
  837. requestID, date, version := getDebugHeaders(resp.Header)
  838. if len(respBody) == 0 {
  839. // no error in response body, might happen in HEAD requests
  840. err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version)
  841. } else {
  842. storageErr := AzureStorageServiceError{
  843. StatusCode: resp.StatusCode,
  844. RequestID: requestID,
  845. Date: date,
  846. APIVersion: version,
  847. }
  848. // response contains storage service error object, unmarshal
  849. if resp.Header.Get("Content-Type") == "application/xml" {
  850. errIn := serviceErrFromXML(respBody, &storageErr)
  851. if err != nil { // error unmarshaling the error response
  852. err = errIn
  853. }
  854. } else {
  855. errIn := serviceErrFromJSON(respBody, &storageErr)
  856. if err != nil { // error unmarshaling the error response
  857. err = errIn
  858. }
  859. }
  860. err = storageErr
  861. }
  862. return err
  863. }