<%SPL::CodeGen::implementationPrologue($model);%>

@include "../pyspltuple.cgt"

<%

#
# Locking
#
# If a window has real-time based policies then window acquire
# data is used as locking. In this case the window object itself
# gets all the locking.
#
# Otherwise if the window has a state handler then an AutoMutex
# is used to provide locking between tuple/punct processing and
# state handler methods.
#
# Otherwise tuple/punct processing is protected by an AutoPortMutex.

# Configure Windowing
 my $inputPort = $model->getInputPortAt(0); 
 my $window = $inputPort->getWindow();
 my $windowCppInitializer = SPL::CodeGen::getWindowCppInitializer($window,"PyObject *");

 # Select the Python wrapper function
 my $pyoutstyle = splpy_tuplestyle($model->getOutputPortAt(0));

 my $out_pywrapfunc=  'object_in__' . $pyoutstyle . '_out';
%>

#define SPLPY_AGGREGATE(f, v, r, occ) \
    streamsx::topology::Splpy::pyTupleMap(f, v, r)

MY_OPERATOR::MY_OPERATOR() :
   funcop_(NULL),
   pyInStyleObj_(NULL),
<%if ($pyoutstyle eq 'dict') {%>    
   pyOutNames_0(NULL),
<%}%>    
   loads(NULL),
   occ_(-1),
   window_(<%=$windowCppInitializer%>),
   _partitionCount(getContext().getMetrics().getCustomMetricByName("nCurrentPartitions"))
   
#if SPLPY_PARTITION_IS_STATEFUL == 1
   , partitionStateHandler_(NULL)
#endif
{
<% if ($window->isSliding()) {%>
    window_.registerOnWindowTriggerHandler(this);
    window_.registerAfterTupleEvictionHandler(this);
<%}%>
<% if ($window->isTumbling()) {%>
    window_.registerBeforeWindowFlushHandler(this);
<%}%>
#if SPLPY_PARTITION_BY_PYTHON == 1
    window_.registerOnWindowPartitionEviction(this);
#endif

#if SPLPY_OP_STATE_HANDLER == 1
    window_.registerSerializationHandler(this);
#endif

    const char * out_wrapfn = "<%=$out_pywrapfunc%>";
<%
# If occ parameter is positive then pass-by-ref is possible
# Generate code to allow pass by ref but only use when
# not connected to a PE output port.

 my $oc = $model->getParameterByName("outputConnections");

 if ($oc) {
    my $occ = $oc->getValueAt(0)->getSPLExpression();
    if ($occ > 0) {
        my $pybyrefwrapfunc = 'object_in__object_out';
%>

#undef SPLPY_AGGREGATE
#define SPLPY_AGGREGATE(f, v, r, occ) \
    streamsx::topology::Splpy::pyTupleMapByRef(f, v, r, occ)

    if (!this->getOutputPortAt(0).isConnectedToAPEOutputPort()) {
       // pass by reference
       out_wrapfn = "<%=$pybyrefwrapfunc%>";
       occ_ = <%=$occ%>;
    }
<%
    } 

    
 }
%>
    _partitionCount.setValueNoLock(0);

    funcop_ = new SplpyFuncOp(this, SPLPY_CALLABLE_STATE_HANDLER, out_wrapfn);

@include "../pyspltuple_constructor.cgt"
    
    // Obtain the function that loads the tuple's value in process()
    {
    SplpyGIL lock;
    <%if ($pyoutstyle eq 'dict') {%>
    pyOutNames_0 = Splpy::pyAttributeNames(getOutputPortAt(0));
    <%}%>    
    <%if ($pystyle eq 'pickle'){%>
    loads = SplpyGeneral::loadFunction("pickle", "loads");
    <% } elsif ($pystyle eq 'json'){ %>
    loads = SplpyGeneral::loadFunction("json", "loads");
    <% } %>
    }
#if SPLPY_OP_STATE_HANDLER == 1
   this->getContext().registerStateHandler(*this);
#endif

#if SPLPY_PARTITION_BY_PYTHON == 1
    // This is based on SplpyFuncOp::loadAndWrapCallable
    SplpyGIL lock;
    setOp(this);

    // The object to be called is either partitionCallable for
    // a function passed into the operator
    // or a pickled encoded class instance
    // represented as a string in parameter pyPartitionByCallable
    
#if SPLPY_PARTITION_BY_CALLABLE == 1

    // argument is the serialized callable instance
    PyObject * partitionCallable = pyUnicode_FromUTF8(param("pyPartitionByCallable").c_str());

    // base64.decode
    PyObject * decoded = SplpyGeneral::callFunction("base64","b64decode", partitionCallable, NULL);

    // dill.loads
    PyObject * loaded = SplpyGeneral::callFunction("dill", "loads", decoded, NULL);
       
    setCallable(loaded);

#else
    
    setCallable( 
        SplpyGeneral::loadFunction(param("pyPartitionByModule"), param("pyPartitionByName")));

#endif // SPLPY_PARTITION_BY_CALLABLE

#if SPLPY_PARTITION_IS_STATEFUL == 1
    partitionStateHandler_ = new SplpyOpStateHandlerImpl(this);
#endif

#endif // SPLPY_PARTITION_BY_PYTHON
}

MY_OPERATOR::~MY_OPERATOR() 
{
  <%if ($pyoutstyle eq 'dict') {%> 
  {
    SplpyGIL lock;
    Py_CLEAR(pyOutNames_0);
  }
  <%}%>  	
  delete funcop_;

#if SPLPY_PARTITION_IS_STATEFUL == 1
    delete partitionStateHandler_;
#endif

  <% if (($pystyle eq 'json') || ($pystyle eq 'pickle')) {%>
  {
      SplpyGIL lock;
      Py_CLEAR(loads);
#if SPLPY_PARTITION_BY_PYTHON == 1 
      clearOp();
      clearCallable();
#endif
  }
  <% } %>
  
}

void MY_OPERATOR::prepareToShutdown() 
{
    funcop_->prepareToShutdown();
}

void MY_OPERATOR::process(Tuple const & tuple, uint32_t port)
{
@include "../pyspltuple2value.cgt"

  PyObject *python_value;

  // If the input style is pickle,

  // None of the streamsx::topology methods in this scope grab the lock
  // so we need to do it here.
  
  <%if ($pystyle eq 'pickle'){%>
      unsigned char const *data = value.getData();
      unsigned char fmt = *data;
      if (fmt == STREAMSX_TPP_PTR) {
          __SPLTuplePyPtr *stp = (__SPLTuplePyPtr *)(data);
          python_value = stp->pyptr;
      }
      // Anything ASCII is also Pickle (Python 2 default format)
      else if (fmt <= STREAMSX_TPP_PICKLE) {
      	  // This is a pickled value. Need to depickle it.
	  {
	      SplpyGIL lock; 
              python_value = pySplValueToPyObject(value);

	      // Depickle the tuple.
	      PyObject *tup = PyTuple_New(1);
	      PyTuple_SET_ITEM(tup, 0, python_value);
   	      python_value = SplpyGeneral::pyCallObject(loads, tup);

	  } // End SplpyGIL lock
      }
  <% } elsif ($pystyle eq 'string'){%>
      {
	  SplpyGIL lock;
          python_value = pySplValueToPyObject(value);
      }
  <% } elsif ($pystyle eq 'json'){%>
      {
      	  SplpyGIL lock;
          python_value = pySplValueToPyObject(value);
	  PyObject *tup = PyTuple_New(1);
	  PyTuple_SET_ITEM(tup, 0, python_value);
	  python_value = SplpyGeneral::pyCallObject(loads, tup);
      }
  <% } elsif ($pystyle eq 'dict' || $pystyle eq 'tuple' || $pystyle_nt) {%>
      python_value = value;
 <% } else{
	  SPL::CodeGen::exitln($pystyle . " is an unsupported input type.");      
    }
 %>


#if SPLPY_AGGREGATE_TIME_POLICIES == 1
   // window_.insert() obtains the mutex.
#elif SPLPY_OP_STATE_HANDLER == 1
    SPL::AutoMutex am(mutex_);
#else
    SPL::AutoPortMutex am(mutex_, *this);
#endif

#if SPLPY_PARTITION_BY_ATTRIBUTE == 1
  PartitionType const & partition = getPartitionValue(static_cast<TupleType const &>(tuple));
  window_.insert(python_value, partition);
#elif SPLPY_PARTITION_BY_PYTHON == 1
  PartitionType const & partition = getPartitionValue(python_value);
  window_.insert(python_value, partition);
#else
  window_.insert(python_value);
#endif

  _partitionCount.setValueNoLock(window_.getWindowStorage().size());
}

void MY_OPERATOR::process(Punctuation const & punct, uint32_t port)
{
<% if ($window->isTumbling()) {%>
   // Aggregate the remaining contents if there are some.
   if (punct == Punctuation::FinalMarker)
       aggregateRemaining();
<%}%>
}

<% if ($window->isTumbling()) {%>
void MY_OPERATOR::aggregateRemaining() {
#if SPLPY_AGGREGATE_TIME_POLICIES == 1
    SPL::AutoWindowDataAcquirer<WindowType::TupleType, WindowType::PartitionType, WindowType::DataType, WindowType::StorageType> awd(window_);
#elif SPLPY_OP_STATE_HANDLER == 1
    SPL::AutoMutex am(mutex_);
#else
    SPL::AutoPortMutex am(mutex_, *this);
#endif
    WindowType::StorageType & storage = window_.getWindowStorage();
    // Iterate through the partitions.
    for (WindowType::StorageType::iterator partition = storage.begin(); partition != storage.end(); ++partition) {
        WindowType::DataType & content = partition->second;
	if (!content.empty()) {
            beforeWindowFlushEvent(window_, partition->first);

           // Since we have processed these tuples in batch
           // don't process them again. 
           content.clear();
        }
    }
}
<%}%>

#if SPLPY_PARTITION_BY_PYTHON == 1
PyObject * MY_OPERATOR::getPartitionValue(PyObject * tuple) const {
    SPLAPPTRC(L_DEBUG, "getPartitionValue: enter", SPLPY_SH_ASPECT);
    SplpyGIL lock; 
    PyObject * arg = PyTuple_New(1);
    Py_INCREF(tuple);
    PyTuple_SET_ITEM(arg, 0, tuple);
    SPLAPPTRC(L_DEBUG, "getPartitionValue: pyCallObject", SPLPY_SH_ASPECT);
    PyObject *ret = SplpyGeneral::pyCallObject(callable(), arg);

    if (!ret) {
        SplpyGeneral::tracePythonError();        
        throw SplpyExceptionInfo::pythonError("partition").exception();
    }

    SPLAPPTRC(L_DEBUG, "getPartitionValue: exit", SPLPY_SH_ASPECT);
    return ret;
}
#endif // SPLPY_PARTITION_BY_PYTHON

// ##############################
// Window Event Handler Overrides
// ##############################

<% if ($window->isSliding()) {%>

void MY_OPERATOR::afterTupleEvictionEvent(
     WindowEventType::WindowType & window,  WindowEventType::TupleType & tuple,  WindowEventType::PartitionType const & partition) {
     // Drop reference to tuple after it is removed from the window.
     SplpyGIL lock;
     Py_DECREF(tuple);
}

<%}%>

// Perform the aggregation.
<% if ($window->isSliding()) {%>
void MY_OPERATOR::onWindowTriggerEvent(
<%}%>
<% if ($window->isTumbling()) {%>
void MY_OPERATOR::beforeWindowFlushEvent(
<%}%>
    WindowEventType::WindowType & window, WindowEventType::PartitionType const & key){    
    SPLAPPTRC(L_DEBUG, "aggregation: enter", SPLPY_SH_ASPECT);
    WindowType::StorageType & storage = window.getWindowStorage();

    WindowType::DataType & content = storage[key];
    PyObject *items;
    {
      SplpyGIL lock;
      items = PyList_New(content.size());
      unsigned int idx = 0;
      for(WindowType::DataType::iterator it=content.begin(); it!=content.end(); ++it) {
        PyObject *item = *it;
<% if ($window->isSliding()) {%>
	// The list steals a reference, increment such that the window can maintain a copy
	// once the tuple is deleted. Woith tumbling the window does
        // retain the contents.
	Py_INCREF(item);
<%}%>
	PyList_SET_ITEM(items, idx, item);
	++idx;
      }
    }
  OPort0Type otuple;

  {
    try {
      SplpyGIL lock;
      SPLAPPTRC(L_DEBUG, "aggregation: call", SPLPY_SH_ASPECT);
      <%if ($pyoutstyle eq 'dict') {%>
	    PyObject * ret = streamsx::topology::Splpy::pyTupleMap(funcop_->callable(), items);
	    if (ret == NULL)
		  return;
	    if (PyTuple_Check(ret)) {
		  fromPyTupleToSPLTuple(ret, otuple);
		  Py_DECREF(ret);
	    } else if (PyDict_Check(ret)) {
		  fromPyDictToSPLTuple(ret, otuple);
		  Py_DECREF(ret);
	    } else {
		  throw SplpyGeneral::generalException("submit",
		   "Fatal error: Value submitted must be a Python tuple or dict.");
	    }
	  SPLAPPTRC(L_DEBUG, "aggregation: submit", SPLPY_SH_ASPECT);
      submit(otuple, 0);
      <% } else { %>
      if (SPLPY_AGGREGATE(funcop_->callable(), items,
        otuple.get_<%=$model->getOutputPortAt(0)->getAttributeAt(0)->getName()%>(), occ_)){
        SPLAPPTRC(L_DEBUG, "aggregation: submit", SPLPY_SH_ASPECT);
        submit(otuple, 0);
      } 
      else {
        SPLAPPTRC(L_DEBUG, "aggregation: fail", SPLPY_SH_ASPECT);
      }	   
      <%}%>
    } catch (const streamsx::topology::SplpyExceptionInfo& excInfo) {
      SPLAPPTRC(L_DEBUG, "aggregation: caught", SPLPY_SH_ASPECT);
      SPLPY_OP_HANDLE_EXCEPTION_INFO_GIL(excInfo);
      return;
    }
    submit(Punctuation::WindowMarker, 0);
  }
  SPLAPPTRC(L_DEBUG, "aggregation: exit", SPLPY_SH_ASPECT);
}

#if SPLPY_PARTITION_BY_PYTHON == 1
void MY_OPERATOR::onWindowPartitionEviction(WindowEventType::WindowType & window, WindowEventType::PartitionIterator begin, WindowEventType::PartitionIterator end) {
  SplpyGIL lock;
  
  for (WindowEventType::PartitionIterator it = begin; it != end; ++it) {
    Py_XDECREF(it->first);
  }
}
#endif

// ##################################
// End Window Event Handler Overrides
// ##################################

#if SPLPY_OP_STATE_HANDLER == 1

/* StateHandler methods */

void MY_OPERATOR::drain() {
    SPLAPPTRC(L_DEBUG, "drain-Aggregate: enter", SPLPY_SH_ASPECT);
<% if ($window->isTumbling()) {%>
   aggregateRemaining();
<%}%>
#if SPLPY_AGGREGATE_TIME_POLICIES == 0
    SPL::AutoMutex am(mutex_);
#endif
    window_.drain();
    SPLAPPTRC(L_DEBUG, "drain-Aggregate: exit", SPLPY_SH_ASPECT);
}

void MY_OPERATOR::checkpoint(SPL::Checkpoint & ckpt) {
    SPLAPPTRC(L_DEBUG, "checkpoint-Aggregate: enter", SPLPY_SH_ASPECT);
#if SPLPY_AGGREGATE_TIME_POLICIES == 0
    SPL::AutoMutex am(mutex_);
#endif
    window_.checkpoint(ckpt);
#if SPLPY_PARTITION_IS_STATEFUL == 1
    try {
        partitionStateHandler_->checkpoint(ckpt);
    }
    catch (SplpyExceptionInfo const & e) {
        throw e.exception();
    }
#endif	
    SPLAPPTRC(L_DEBUG, "checkpoint-Aggregate: exit", SPLPY_SH_ASPECT);
}

void MY_OPERATOR::reset(SPL::Checkpoint & ckpt) {
    SPLAPPTRC(L_DEBUG, "reset-Aggregate: enter", SPLPY_SH_ASPECT);
#if SPLPY_AGGREGATE_TIME_POLICIES == 0
    SPL::AutoMutex am(mutex_);
#endif
    window_.reset(ckpt);
#if SPLPY_PARTITION_IS_STATEFUL == 1
    try {
        partitionStateHandler_->reset(ckpt);
    }
    catch (SplpyExceptionInfo const & e) {
        throw e.exception();
    }
#endif
    SPLAPPTRC(L_DEBUG, "reset-Aggregate: exit", SPLPY_SH_ASPECT);
}

void MY_OPERATOR::resetToInitialState() {
    SPLAPPTRC(L_DEBUG, "resetToInitialState-Aggregate: enter", SPLPY_SH_ASPECT);
#if SPLPY_AGGREGATE_TIME_POLICIES == 0
    SPL::AutoMutex am(mutex_);
#endif
    window_.resetToInitialState();
#if SPLPY_PARTITION_IS_STATEFUL == 1
    try {
        partitionStateHandler_->resetToInitialState();
    }
    catch (SplpyExceptionInfo const & e) {
        throw e.exception();
    }
#endif
    SPLAPPTRC(L_DEBUG, "resetToInitialState-Aggregate: exit", SPLPY_SH_ASPECT);
}

#endif

#if SPLPY_PARTITION_BY_PYTHON == 1
  void MY_OPERATOR::setCallable(PyObject * callable) {
    SPLAPPTRC(L_DEBUG, "setCallable: enter", SPLPY_SH_ASPECT);
    Py_CLEAR(callable_);
    callable_ = callable;
    Py_INCREF(callable);
    SPLAPPTRC(L_DEBUG, "setCallable: exit", SPLPY_SH_ASPECT);
  } 
  void MY_OPERATOR::clearCallable() {
    SPLAPPTRC(L_DEBUG, "clearCallable: enter", SPLPY_SH_ASPECT);
    Py_CLEAR(callable_);
    SPLAPPTRC(L_DEBUG, "clearCallable: exit", SPLPY_SH_ASPECT);
  }

#endif

<%
  if ($pyoutstyle eq 'dict') {
	my %cpp_tuple_types;  
    # In this case we don't want the function that
    # converts the Python tuple to an SPL tuple to
    # copy attributes from the input port
    my $iport;

    my $oport = $model->getOutputPortAt(0);
    my $oport_submission = 0;
    my $otupleType = $oport->getSPLTupleType();
    my @onames = SPL::CodeGen::Type::getAttributeNames($otupleType);
    my @otypes = SPL::CodeGen::Type::getAttributeTypes($otupleType);

%>
// Create member function that converts Python tuple to SPL tuple
@include  "../../opt/python/codegen/py_pyTupleTosplTuple.cgt"
<%}%>


<%SPL::CodeGen::implementationEpilogue($model);%>
