class_docstr: Handles Amazon SageMaker processing tasks for jobs using PySpark.
class_docstr_tok: ["Handles","Amazon","SageMaker","processing","tasks","for","jobs","using","PySpark","."]
docstr: Runs a processing job.
  
  Args:
      submit_app (str): Path (local or S3) to Python file to submit to Spark
          as the primary application
      submit_py_files (list[str] or list[PipelineVariable]): List of paths (local or S3)
          to provide for `spark-submit --py-files` option
      submit_jars (list[str] or list[PipelineVariable]): List of paths (local or S3)
          to provide for `spark-submit --jars` option
      submit_files (list[str] or list[PipelineVariable]): List of paths (local or S3)
          to provide for `spark-submit --files` option
      inputs (list[:class:`~sagemaker.processing.ProcessingInput`]): Input files for
          the processing job. These must be provided as
          :class:`~sagemaker.processing.ProcessingInput` objects (default: None).
      outputs (list[:class:`~sagemaker.processing.ProcessingOutput`]): Outputs for
          the processing job. These can be specified as either path strings or
          :class:`~sagemaker.processing.ProcessingOutput` objects (default: None).
      arguments (list[str] or list[PipelineVariable]): A list of string arguments to
          be passed to a processing job (default: None).
      wait (bool): Whether the call should wait until the job completes (default: True).
      logs (bool): Whether to show the logs produced by the job.
          Only meaningful when wait is True (default: True).
      job_name (str): Processing job name. If not specified, the processor generates
          a default job name, based on the base job name and current timestamp.
      experiment_config (dict[str, str]): Experiment management configuration.
          Optionally, the dict can contain three keys:
          'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
          The behavior of setting these keys is as follows:
          * If `ExperimentName` is supplied but `TrialName` is not a Trial will be
          automatically created and the job's Trial Component associated with the Trial.
          * If `TrialName` is supplied and the Trial already exists the job's Trial Component
          will be associated with the Trial.
          * If both `ExperimentName` and `TrialName` are not supplied the trial component
          will be unassociated.
          * `TrialComponentDisplayName` is used for display in Studio.
      configuration (list[dict] or dict): Configuration for Hadoop, Spark, or Hive.
          List or dictionary of EMR-style classifications.
          https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html
      spark_event_logs_s3_uri (str or PipelineVariable): S3 path where spark application
          events will be published to.
      kms_key (str): The ARN of the KMS key that is used to encrypt the
          user code file (default: None).
docstr_tok: ["Runs","a","processing","job",".","Args",":","submit_app","(","str",")",":","Path","(","local","or","S3",")","to","Python","file","to","submit","to","Spark","as","the","primary","application","submit_py_files","(","list","[","str","]","or","list","[","PipelineVariable","]",")",":","List","of","paths","(","local","or","S3",")","to","provide","for","`","spark-submit","--","py-files","`","option","submit_jars","(","list","[","str","]","or","list","[","PipelineVariable","]",")",":","List","of","paths","(","local","or","S3",")","to","provide","for","`","spark-submit","--","jars","`","option","submit_files","(","list","[","str","]","or","list","[","PipelineVariable","]",")",":","List","of","paths","(","local","or","S3",")","to","provide","for","`","spark-submit","--","files","`","option","inputs","(","list","[",":","class",":","`","~sagemaker.processing.ProcessingInput","`","]",")",":","Input","files","for","the","processing","job",".","These","must","be","provided","as",":","class",":","`","~sagemaker.processing.ProcessingInput","`","objects","(","default",":","None",")",".","outputs","(","list","[",":","class",":","`","~sagemaker.processing.ProcessingOutput","`","]",")",":","Outputs","for","the","processing","job",".","These","can","be","specified","as","either","path","strings","or",":","class",":","`","~sagemaker.processing.ProcessingOutput","`","objects","(","default",":","None",")",".","arguments","(","list","[","str","]","or","list","[","PipelineVariable","]",")",":","A","list","of","string","arguments","to","be","passed","to","a","processing","job","(","default",":","None",")",".","wait","(","bool",")",":","Whether","the","call","should","wait","until","the","job","completes","(","default",":","True",")",".","logs","(","bool",")",":","Whether","to","show","the","logs","produced","by","the","job",".","Only","meaningful","when","wait","is","True","(","default",":","True",")",".","job_name","(","str",")",":","Processing","job","name",".","If","not","specified",",","the","processor","generates","a","default","job","name",",","based","on","the","base","job","name","and","current","timestamp",".","experiment_config","(","dict","[","str",",","str","]",")",":","Experiment","management","configuration",".","Optionally",",","the","dict","can","contain","three","keys",":","'ExperimentName","'",",","'TrialName","'",",","and","'TrialComponentDisplayName","'",".","The","behavior","of","setting","these","keys","is","as","follows",":","*","If","`","ExperimentName","`","is","supplied","but","`","TrialName","`","is","not","a","Trial","will","be","automatically","created","and","the","job","'s","Trial","Component","associated","with","the","Trial",".","*","If","`","TrialName","`","is","supplied","and","the","Trial","already","exists","the","job","'s","Trial","Component","will","be","associated","with","the","Trial",".","*","If","both","`","ExperimentName","`","and","`","TrialName","`","are","not","supplied","the","trial","component","will","be","unassociated",".","*","`","TrialComponentDisplayName","`","is","used","for","display","in","Studio",".","configuration","(","list","[","dict","]","or","dict",")",":","Configuration","for","Hadoop",",","Spark",",","or","Hive",".","List","or","dictionary","of","EMR-style","classifications",".","https",":","\/\/docs.aws.amazon.com\/emr\/latest\/ReleaseGuide\/emr-configure-apps.html","spark_event_logs_s3_uri","(","str","or","PipelineVariable",")",":","S3","path","where","spark","application","events","will","be","published","to",".","kms_key","(","str",")",":","The","ARN","of","the","KMS","key","that","is","used","to","encrypt","the","user","code","file","(","default",":","None",")","."]
code: def run(
      self,
      submit_app: str,
      submit_py_files: Optional[
          List[Union[str, PipelineVariable]]
      ] = None,
      submit_jars: Optional[List[Union[str, PipelineVariable]]] = None,
      submit_files: Optional[List[Union[str, PipelineVariable]]] = None,
      inputs: Optional[List[ProcessingInput]] = None,
      outputs: Optional[List[ProcessingOutput]] = None,
      arguments: Optional[List[Union[str, PipelineVariable]]] = None,
      wait: bool = True,
      logs: bool = True,
      job_name: Optional[str] = None,
      experiment_config: Optional[Dict[str, str]] = None,
      configuration: Optional[Union[List[Dict], Dict]] = None,
      spark_event_logs_s3_uri: Optional[
          Union[str, PipelineVariable]
      ] = None,
      kms_key: Optional[str] = None,
  ):
      """Runs a processing job.
  
      Args:
          submit_app (str): Path (local or S3) to Python file to submit to Spark
              as the primary application
          submit_py_files (list[str] or list[PipelineVariable]): List of paths (local or S3)
              to provide for `spark-submit --py-files` option
          submit_jars (list[str] or list[PipelineVariable]): List of paths (local or S3)
              to provide for `spark-submit --jars` option
          submit_files (list[str] or list[PipelineVariable]): List of paths (local or S3)
              to provide for `spark-submit --files` option
          inputs (list[:class:`~sagemaker.processing.ProcessingInput`]): Input files for
              the processing job. These must be provided as
              :class:`~sagemaker.processing.ProcessingInput` objects (default: None).
          outputs (list[:class:`~sagemaker.processing.ProcessingOutput`]): Outputs for
              the processing job. These can be specified as either path strings or
              :class:`~sagemaker.processing.ProcessingOutput` objects (default: None).
          arguments (list[str] or list[PipelineVariable]): A list of string arguments to
              be passed to a processing job (default: None).
          wait (bool): Whether the call should wait until the job completes (default: True).
          logs (bool): Whether to show the logs produced by the job.
              Only meaningful when wait is True (default: True).
          job_name (str): Processing job name. If not specified, the processor generates
              a default job name, based on the base job name and current timestamp.
          experiment_config (dict[str, str]): Experiment management configuration.
              Optionally, the dict can contain three keys:
              'ExperimentName', 'TrialName', and 'TrialComponentDisplayName'.
              The behavior of setting these keys is as follows:
              * If `ExperimentName` is supplied but `TrialName` is not a Trial will be
              automatically created and the job's Trial Component associated with the Trial.
              * If `TrialName` is supplied and the Trial already exists the job's Trial Component
              will be associated with the Trial.
              * If both `ExperimentName` and `TrialName` are not supplied the trial component
              will be unassociated.
              * `TrialComponentDisplayName` is used for display in Studio.
          configuration (list[dict] or dict): Configuration for Hadoop, Spark, or Hive.
              List or dictionary of EMR-style classifications.
              https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html
          spark_event_logs_s3_uri (str or PipelineVariable): S3 path where spark application
              events will be published to.
          kms_key (str): The ARN of the KMS key that is used to encrypt the
              user code file (default: None).
      """
      self._current_job_name = self._generate_current_job_name(
          job_name=job_name
      )
  
      if not submit_app:
          raise ValueError("submit_app is required")
  
      extended_inputs, extended_outputs = self._extend_processing_args(
          inputs=inputs,
          outputs=outputs,
          submit_py_files=submit_py_files,
          submit_jars=submit_jars,
          submit_files=submit_files,
          configuration=configuration,
          spark_event_logs_s3_uri=spark_event_logs_s3_uri,
      )
  
      return super().run(
          submit_app=submit_app,
          inputs=extended_inputs,
          outputs=extended_outputs,
          arguments=arguments,
          wait=wait,
          logs=logs,
          job_name=self._current_job_name,
          experiment_config=experiment_config,
          kms_key=kms_key,
      )
  
imports: from __future__ import absolute_import
  import json
  import logging
  import os.path
  import shutil
  import subprocess
  import tempfile
  import time
  import urllib.request
  from enum import Enum
  from io import BytesIO
  from urllib.parse import urlparse
  from copy import copy
  from typing import Union, List, Dict, Optional
  from sagemaker import image_uris, s3
  from sagemaker.local.image import _ecr_login_if_needed, _pull_image
  from sagemaker.processing import ProcessingInput, ProcessingOutput, ScriptProcessor
  from sagemaker.s3 import S3Uploader
  from sagemaker.session import Session
  from sagemaker.network import NetworkConfig
  from sagemaker.spark import defaults
  from sagemaker.utils import format_tags, Tags
  from sagemaker.workflow import is_pipeline_variable
  from sagemaker.workflow.pipeline_context import runnable_by_pipeline
  from sagemaker.workflow.entities import PipelineVariable
  from sagemaker.workflow.functions import Join
label_desc: Use image node_id 3 for calling the PySparkProcessor obj's underlying member method code with example usage: obj.run(submit_app, submit_py_files, submit_jars, submit_files, inputs, outputs, arguments, wait, logs, job_name, experiment_config, configuration, spark_event_logs_s3_uri, kms_key) and returns: super