fbpx
POSThttps://api.spotinst.io/aws/emr/mrScaler?accountId={ACCOUNT_ID}

Create a new EMR cluster

URL Parameters

Parameter Type Description
ACCOUNT_ID String

Your Spotinst accountId associated with your token

Example: act-123abc
Request

Headers
{
  "Content-Type": "application/json",
  "Authorization": "Bearer ${token}"
}
Body
Copied!
Downloaded!
{
  "mrScaler": {
    "name": "my MRScaler",
    "description": "Spotinst MRScaler",
    "region": "us-west-2",
    "strategy": {
      "cloning": {
        "originClusterId": "j-38EE27G2QY02I",
        "includeSteps":false
      }
    },
    "compute": {
      "availabilityZones": [
        {
          "name": "us-west-2a",
          "subnetId": "subnet-3b5b3601"
        },
        {
          "name": "us-west-2b",
          "subnetId": "subnet-6b5c4209"
        }
      ],
      "bootstrapActions": {
        "file": {
          "bucket": "emr-test",
          "key": "MyFile.json"
        }
      },  
      "instanceGroups": {
        "masterGroup": {
          "instanceTypes": [
            "m3.xlarge",
            "m4.large",
            "m4.xlarge",
            "m4.2xlarge"
          ],
          "target": 1,
          "lifeCycle": "ON_DEMAND"
        },
        "coreGroup": {
          "instanceTypes": [
            "m3.xlarge",
            "m4.large",
            "m4.xlarge",
            "m4.2xlarge"
          ],
          "target": 5,
          "lifeCycle": "SPOT",
          "ebsConfiguration": {
            "ebsBlockDeviceConfigs": [
              {
                "volumeSpecification": {
                  "volumeType": "io1",
                  "sizeInGB": 8,
                  "iops": 200
                },
                "volumesPerInstance": 1
              }
            ],
            "ebsOptimized": true
          }
        },
        "taskGroup": {
          "instanceTypes": [
            "m1.medium",
            "c3.xlarge",
            "m3.xlarge"
          ],
          "capacity": {
            "minimum": 0,
            "maximum": 30,
            "target": 15
          },
          "lifeCycle": "SPOT",
          "ebsConfiguration": {
            "ebsBlockDeviceConfigs": [
              {
                "volumeSpecification": {
                  "volumeType": "gp2",
                  "sizeInGB": 8
                },
                "volumesPerInstance": 1
              },
              {
                "volumeSpecification": {
                  "volumeType": "io1",
                  "sizeInGB": 32
                },
                "volumesPerInstance": 2
              }
            ],
            "ebsOptimized": true
          }
        }
      },
      "configurations": {
        "file": {
          "bucket": "spotinst-labs",
          "key": "emr_configurations.json"
        }
      }
    },
    "scaling": {
      "up": [
        {
          "metricName": "AppsPending",
          "statistic": "average",
          "unit": "count",
          "threshold": 100,
          "actionType": "adjustment",
          "adjustment": 2,
          "namespace": "AWS/ElasticMapReduce",
          "period": 300,
          "evaluationPeriods": 1,
          "cooldown": 600,
          "dimensions": [
            {
              "name": "JobFlowId"
            }
          ],
          "operator": "gte"
        }
      ],
      "down": [
        {
          "metricName": "AppsPending",
          "statistic": "average",
          "unit": "count",
          "threshold": 10,
          "actionType": "adjustment",
          "adjustment": 2,
          "namespace": "AWS/ElasticMapReduce",
          "period": 300,
          "evaluationPeriods": 1,
          "cooldown": 600,
          "dimensions": [
            {
              "name": "JobFlowId"
            }
          ],
          "operator": "lte"
        }
      ]
    }
  }
}

Body Attributes


  • name * String
    MRScaler name
  • description String
    Describe your Mr Scaler
  • region * String
    The region of the source cluster
  • strategy.wrapping * Object
    (Required. Unless strategy.cloning strategy is specified) In wrap mode, MRScaler will manage an existing cluster and will scale up/down cluster task groups only. MRScaler will manage only instance groups that were created by Spotinst
  • strategy.wrapping.sourceClusterId * String
    (Required in wrap strategy only) The id of the cluster to wrap
  • strategy.cloning * Object
    (Required. Unless strategy.wrapping strategy is specified) In clone mode, MRScaler will create a new cluster that will be copied from the origin cluster. MRScaler will manage the entire cluster (the origin cluster will not be affected)
  • strategy.cloning.originClusterId * Boolean
    (Required in clone strategy only) The id of the cluster to clone (required in clone strategy only)
  • strategy.cloning.includeSteps * Boolean
    (Required in clone strategy only) include cloning of steps from the original cluster or not
  • strategy.availabilityZones * Array<String>
    (Required) Information about one or more availability Zones for the cluster (required in clone strategy only)
  • compute.ebsRootVolumeSize Integer
    The EBS root volume size in GB
  • compute.availabilityZones.name * String
    The availability zone name (required in clone strategy only)
  • compute.availabilityZones.subnetId * String
    (Required) Specify EC2 \ VPC subnet id (required in clone strategy only)
  • compute.bootstrapActions Object
    Describe bootstrap action. for more information please see: https://help.spotinst.com/hc/en-us/articles/360000322509
    Example: "bootstrapActions": { "file": { "bucket": "emr-test", "key": "MyFile.json" } }
  • compute.bootstrapActions.file Object
    Describe the bootstrap file information
  • compute.bootstrapActions.file.bucket String
    The S3 Bucket name
  • compute.bootstrapActions.file.key String
    The S3 Bucket key
  • compute.instanceGroups.masterGroup.instanceTypes * Array<String>
    The types to choose from for master group (required in clone strategy only)
  • compute.instanceGroups.masterGroup.lifeCycle * String
    The life cycle of master group. Possible values: SPOT, ON_DEMAND
  • compute.instanceGroups.masterGroup.ebsConfiguration Object
    Information about the EBS configurations that will be attached to each EC2 instance in the instance group.
  • compute.instanceGroups.masterGroup.target * Integer
    Number of instances in the master group (always set to 1) (required in clone strategy only)
  • compute.instanceGroups.coreGroup.instanceTypes * Array<String>
    (Required in clone strategy only) The types to choose from for core group
  • compute.instanceGroups.coreGroup.lifeCycle * String
    (Required in clone strategy only) The life cycle of core group. Possible values: SPOT, ON_DEMAND
  • compute.instanceGroups.coreGroup.target * Integer
    (Required in clone strategy only) Number of instances in the core group
  • compute.instanceGroups.taskGroup.ebsConfiguration Object
    (Optional) information about the EBS configurations that will be attached to each EC2 instance in the instance group.
  • compute.instanceGroups.taskGroup.instanceTypes * Array<String>
    The types to choose from for task group
  • compute.instanceGroups.coreGroup.capacity.target * Integer
    Number of desired instances in task groups
  • compute.instanceGroups.coreGroup.capacity.minimum * Integer
    The lower limit of instances in task groups
  • compute.instanceGroups.coreGroup.capacity.maximum * Integer
    The upper limit of instances in task groups
  • compute.instanceGroups.coreGroup.ebsConfiguration Object
    Information about the EBS configurations that will be attached to each EC2 instance in the instance group.
  • compute.configurations Object
    Information about the configurations file that will be used in the EMR cluster. for more information please see: https://help.spotinst.com/hc/en-us/articles/360000322509
  • compute.configurations.file * Object
    Location and name of the configurations file that will be used in the EMR cluster.
  • compute.configurations.file.bucket * String
    The S3 bucket where the file is stored.
  • compute.configurations.file.key * String
    The key of the file.
  • ebsConfiguration.ebsOptimized Boolean
    Indicates whether an Amazon EBS volume is EBS-optimized.
  • ebsConfiguration.ebsConfigs Array<Object>
    An array of Amazon EBS volume specifications attached to a cluster instance.
  • ebsConfiguration.ebsConfigs.VolumesPerInstance * Integer
    Number of EBS volumes with a specific volume configuration that will be associated with every instance in the instance group (Required if ebsconfigs is defined)
  • ebsConfiguration.ebsConfigs.VolumeSpecification * Object
    EBS volume specifications such as volume type, IOPS, and size (GiB) that will be requested for the EBS volume attached to an EC2 instance in the cluster (required if ebsConfigs is defined).
  • ebsConfiguration.ebsConfigs.VolumeSpecification.VolumeType * String
    The volume type. Volume types supported are gp2, io1, standard (required if ebsConfigs is defined).
  • ebsConfiguration.ebsConfigs.VolumeSpecification.Iops Integer
    The number of I/O operations per second (IOPS) that the volume supports.
  • ebsConfiguration.ebsConfigs.VolumeSpecification.SizeInGB * Integer
    The volume size, in gibibytes (GiB). This can be a number from 1 - 1024. If the volume type is EBS-optimized, the minimum value is 10 (Required if ebsConfigs is defined).
  • scaling.up.policyName String
    The Scaling policy name
  • scaling.up.metricName String
    The name of the metric
    Default: AppsPending
  • scaling.up.statistic String
    The metric statistics to return. Valid Values: average, sum, sampleCount, maximum, minimum
  • scaling.up.unit String
    The unit for the alarm’s associated metric. Valid Values: seconds | microseconds | milliseconds | bytes | kilobytes | megabytes | gigabytes | terabytes | bits | kilobits | megabits | gigabits | terabits | percent | count | bytes/second | kilobytes/second | megabytes/second | gigabytes/second | terabytes/second | bits/second | kilobits/second | megabits/second | gigabits/second | terabits/second | count/second | none
  • scaling.up.threshold Double
    The value against which the specified statistic is compared.
  • scaling.up.action Object
    The action to take when scale up is needed.
  • scaling.up.action.type String
    The type of the action to take when scale up is needed. Valid Values: adjustment | updateCapacity | setMinTarget
  • scaling.up.action.adjustment Integer
    The number associated with the specified adjustment type. Required if using “adjustment” as action type.
  • scaling.up.action.minTargetCapacity Integer
    The number with the target capacity needed. Required if using “setMinTarget” as action type.
  • scaling.up.action.target Integer
    The desired number of instances. Optional, required if using “updateCapacity” as action type and neither “minimum” nor “maximum” are not defined.
  • scaling.up.action.minimum Integer
    The lower limit number of instances that you can scale down to. Optional, required if using “updateCapacity” as action type and neither “target” nor “maximum” are not defined.
  • scaling.up.action.maximum Integer
    The upper limit number of instances that you can scale up to. Required if using “updateCapacity” as action type and neither “target” nor “minimum” are not defined.
  • scaling.up.namespace String
    The namespace for the alarm’s associated metric.
    Default: AWS/ElasticMapReduce
  • scaling.up.dimensions Array<String>
    The dimensions for the alarm’s associated metric.
  • scaling.up.period Integer
    The period in seconds over which the statistic is applied.
  • scaling.up.evaluationPeriods Integer
    The number of periods over which data is compared to the specified threshold.
  • scaling.up.cooldown Integer
    The amount of time, in seconds, after a scaling activity completes before any further trigger-related scaling activities can start.
  • scaling.up.operator String
    The operator to use in order to determine if the scaling policy is applicable. Valid values: gt | gte | lt | lte
  • scaling.down.policyName String
    The scaling policy name
  • scaling.down.metricName String
    The name of the metric.
    Default: AppsPending
  • scaling.down.statistic String
    The metric statistics to return. Valid Values: average | sum | sampleCount | maximum | minimum
  • scaling.down.unit String
    The unit for the alarm’s associated metric. Valid Values: seconds | microseconds | milliseconds | bytes | kilobytes | megabytes | gigabytes | terabytes | bits | kilobits | megabits | gigabits | terabits | percent | count | bytes/second | kilobytes/second | megabytes/second | gigabytes/second | terabytes/second | bits/second | kilobits/second | megabits/second | gigabits/second | terabits/second | count/second | none
  • scaling.down.action Object
    The action to take when scale down is needed.
  • scaling.down.action.type String
    he type of the action to take when scale down is needed. Valid Values: adjustment | updateCapacity | setMaxTarget
  • scaling.down.action.adjustment Integer
    The number associated with the specified adjustment type. Required if using “adjustment” as action type.
  • scaling.down.action.maxTargetCapacity Integer
    The number with the target capacity needed. Required if using “setMaxTarget” as action type.
  • scaling.down.action.target Integer
    The desired number of instances. Optional, required if using “updateCapacity” as action type and neither “minimum” nor “maximum” are not defined.
  • scaling.down.action.minimum Integer
    The lower limit number of instances that you can scale down to. Required if using “updateCapacity” as action type and neither “target” nor “maximum” are not defined.
  • scaling.down.action.maximum Integer
    The upper limit number of instances that you can scale up to. Required if using “updateCapacity” as action type and neither “target” nor “minimum” are not defined.
  • scaling.down.threshold Double
    The value against which the specified statistic is compared.
  • scaling.down.namespace String
    The namespace for the alarm’s associated metric.
    Default: AWS/ElasticMapReduce
  • scaling.down.dimensions Array<Object>
    The dimensions for the alarm’s associated metric
  • scaling.down.period Integer
    The period in seconds over which the statistic is applied.
  • scaling.down.evaluationPeriods Integer
    The number of periods over which data is compared to the specified threshold
  • scaling.down.cooldown Integer
    The amount of time, in seconds, after a scaling activity completes before any further trigger-related scaling activities can start
  • scaling.down.operator String
    The operator to use in order to determine if the scaling policy is applicable. Valid values: gt | gte | lt | lte
Response - 200

Headers
{
  "Content-Type": "application/json"
}
Body
Copied!
Downloaded!
{
  "request": {
    "id": "24b3159e-e2d9-49bc-bfe2-cc383febd04d",
    "url": "/aws/emr/mrScaler",
    "method": "POST",
    "timestamp": "2016-03-03T19:13:24.062Z"
  },
  "response": {
    "status": {
      "code": 200,
      "message": "OK"
    },
    "kind": "spotinst:aws:emr:mrScaler",
    "items": [
      {
        "id": "simrs-12345",
        "name": "my MRScaler",
        "description": "Spotinst MRScaler",
        "region": "us-west-2",
        "strategy": {
          "cloning": {
            "originClusterId": "j-38EE27G2QY02I"
          }
        },
        "compute": {
          "availabilityZones": [
            {
              "name": "us-west-2a",
              "subnetId": "subnet-3b5b3601"
            },
            {
              "name": "us-west-2b",
              "subnetId": "subnet-6b5c4209"
            }
          ],
          "instanceGroups": {
            "masterGroup": {
              "instanceTypes": [
                "m3.xlarge",
                "m4.large",
                "m4.xlarge",
                "m4.2xlarge"
              ],
              "target": 1,
              "lifeCycle": "ON_DEMAND"
            },
            "coreGroup": {
              "instanceTypes": [
                "m3.xlarge",
                "m4.large",
                "m4.xlarge",
                "m4.2xlarge"
              ],
              "target": 5,
              "lifeCycle": "SPOT"
            },
            "taskGroup": {
              "instanceTypes": [
                "m1.medium",
                "c3.xlarge",
                "m3.xlarge"
              ],
              "capacity": {
                "minimum": 0,
                "maximum": 30,
                "target": 15
              },
              "lifeCycle": "SPOT"
            }
          },
          "configurations": {
            "file": {
              "bucket": "spotinst-labs",
              "key": "emr_configurations.json"
            }
          }
        },
        "scaling": {
          "up": [
            {
              "metricName": "AppsPending",
              "statistic": "average",
              "unit": "count",
              "threshold": 100,
              "minTargetCapacity": 20,
              "namespace": "AWS/ElasticMapReduce",
              "period": 300,
              "evaluationPeriods": 1,
              "cooldown": 600,
              "dimensions": [
                {
                  "name": "JobFlowId"
                }
              ],
              "operator": "gte"
            }
          ],
          "down": [
            {
              "metricName": "AppsPending",
              "statistic": "average",
              "unit": "count",
              "threshold": 10,
              "maxTargetCapacity": 5,
              "namespace": "AWS/ElasticMapReduce",
              "period": 300,
              "evaluationPeriods": 1,
              "cooldown": 600,
              "dimensions": [
                {
                  "name": "JobFlowId"
                }
              ],
              "operator": "lte"
            }
          ]
        }
      }
    ],
    "count": 1
  }
}