1273 lines
41 KiB
C++
1273 lines
41 KiB
C++
#ifndef GREENLET_GREENLET_HPP
|
|
#define GREENLET_GREENLET_HPP
|
|
/*
|
|
* Declarations of the core data structures.
|
|
*/
|
|
|
|
#define PY_SSIZE_T_CLEAN
|
|
#include <Python.h>
|
|
|
|
#include "greenlet_compiler_compat.hpp"
|
|
#include "greenlet_refs.hpp"
|
|
#include "greenlet_cpython_compat.hpp"
|
|
#include "greenlet_allocator.hpp"
|
|
|
|
using greenlet::refs::OwnedObject;
|
|
using greenlet::refs::OwnedGreenlet;
|
|
using greenlet::refs::OwnedMainGreenlet;
|
|
using greenlet::refs::BorrowedGreenlet;
|
|
|
|
#if PY_VERSION_HEX < 0x30B00A6
|
|
# define _PyCFrame CFrame
|
|
# define _PyInterpreterFrame _interpreter_frame
|
|
#endif
|
|
|
|
// XXX: TODO: Work to remove all virtual functions
|
|
// for speed of calling and size of objects (no vtable).
|
|
// One pattern is the Curiously Recurring Template
|
|
namespace greenlet
|
|
{
|
|
class ExceptionState
|
|
{
|
|
private:
|
|
G_NO_COPIES_OF_CLS(ExceptionState);
|
|
|
|
#if PY_VERSION_HEX >= 0x030700A3
|
|
// Even though these are borrowed objects, we actually own
|
|
// them, when they're not null.
|
|
// XXX: Express that in the API.
|
|
private:
|
|
_PyErr_StackItem* exc_info;
|
|
_PyErr_StackItem exc_state;
|
|
#else
|
|
OwnedObject exc_value;
|
|
#if !GREENLET_PY311
|
|
OwnedObject exc_type;
|
|
OwnedObject exc_traceback;
|
|
#endif
|
|
#endif
|
|
public:
|
|
ExceptionState();
|
|
void operator<<(const PyThreadState *const tstate) G_NOEXCEPT;
|
|
void operator>>(PyThreadState* tstate) G_NOEXCEPT;
|
|
void clear() G_NOEXCEPT;
|
|
|
|
int tp_traverse(visitproc visit, void* arg) G_NOEXCEPT;
|
|
void tp_clear() G_NOEXCEPT;
|
|
};
|
|
|
|
template<typename T>
|
|
void operator<<(const PyThreadState *const tstate, T& exc);
|
|
|
|
template<typename IsPy37>
|
|
class PythonStateContext
|
|
{};
|
|
|
|
template<>
|
|
class PythonStateContext<GREENLET_WHEN_PY37>
|
|
{
|
|
protected:
|
|
greenlet::refs::OwnedContext _context;
|
|
public:
|
|
inline const greenlet::refs::OwnedContext& context() const
|
|
{
|
|
return this->_context;
|
|
}
|
|
inline greenlet::refs::OwnedContext& context()
|
|
{
|
|
return this->_context;
|
|
}
|
|
|
|
inline void tp_clear()
|
|
{
|
|
this->_context.CLEAR();
|
|
}
|
|
|
|
template<typename T>
|
|
inline static PyObject* context(T* tstate)
|
|
{
|
|
return tstate->context;
|
|
}
|
|
|
|
template<typename T>
|
|
inline static void context(T* tstate, PyObject* new_context)
|
|
{
|
|
tstate->context = new_context;
|
|
tstate->context_ver++;
|
|
}
|
|
};
|
|
|
|
|
|
template<>
|
|
class PythonStateContext<GREENLET_WHEN_NOT_PY37>
|
|
{
|
|
public:
|
|
inline const greenlet::refs::OwnedContext& context() const
|
|
{
|
|
throw AttributeError("no context");
|
|
}
|
|
|
|
inline greenlet::refs::OwnedContext& context()
|
|
{
|
|
throw AttributeError("no context");
|
|
}
|
|
|
|
inline void tp_clear(){};
|
|
|
|
template<typename T>
|
|
inline static PyObject* context(T* UNUSED(tstate))
|
|
{
|
|
throw PyFatalError("This should never be called.");
|
|
}
|
|
|
|
template<typename T>
|
|
inline static void context(T* UNUSED(tstate), PyObject* UNUSED(new_context))
|
|
{
|
|
throw PyFatalError("This should never be called.");
|
|
}
|
|
};
|
|
|
|
class PythonState : public PythonStateContext<G_IS_PY37>
|
|
{
|
|
public:
|
|
typedef greenlet::refs::OwnedReference<struct _frame> OwnedFrame;
|
|
private:
|
|
G_NO_COPIES_OF_CLS(PythonState);
|
|
// We own this if we're suspended (although currently we don't
|
|
// tp_traverse into it; that's a TODO). If we're running, it's
|
|
// empty. If we get deallocated and *still* have a frame, it
|
|
// won't be reachable from the place that normally decref's
|
|
// it, so we need to do it (hence owning it).
|
|
OwnedFrame _top_frame;
|
|
#if GREENLET_USE_CFRAME
|
|
_PyCFrame* cframe;
|
|
int use_tracing;
|
|
#endif
|
|
int recursion_depth;
|
|
int trash_delete_nesting;
|
|
#if GREENLET_PY311
|
|
_PyInterpreterFrame* current_frame;
|
|
_PyStackChunk* datastack_chunk;
|
|
PyObject** datastack_top;
|
|
PyObject** datastack_limit;
|
|
#endif
|
|
|
|
public:
|
|
PythonState();
|
|
// You can use this for testing whether we have a frame
|
|
// or not. It returns const so they can't modify it.
|
|
const OwnedFrame& top_frame() const G_NOEXCEPT;
|
|
|
|
|
|
void operator<<(const PyThreadState *const tstate) G_NOEXCEPT;
|
|
void operator>>(PyThreadState* tstate) G_NOEXCEPT;
|
|
void clear() G_NOEXCEPT;
|
|
|
|
int tp_traverse(visitproc visit, void* arg, bool visit_top_frame) G_NOEXCEPT;
|
|
void tp_clear(bool own_top_frame) G_NOEXCEPT;
|
|
void set_initial_state(const PyThreadState* const tstate) G_NOEXCEPT;
|
|
#if GREENLET_USE_CFRAME
|
|
void set_new_cframe(_PyCFrame& frame) G_NOEXCEPT;
|
|
#endif
|
|
void will_switch_from(PyThreadState *const origin_tstate) G_NOEXCEPT;
|
|
void did_finish(PyThreadState* tstate) G_NOEXCEPT;
|
|
};
|
|
|
|
class StackState
|
|
{
|
|
// By having only plain C (POD) members, no virtual functions
|
|
// or bases, we get a trivial assignment operator generated
|
|
// for us. However, that's not safe since we do manage memory.
|
|
// So we declare an assignment operator that only works if we
|
|
// don't have any memory allocated. (We don't use
|
|
// std::shared_ptr for reference counting just to keep this
|
|
// object small)
|
|
private:
|
|
char* _stack_start;
|
|
char* stack_stop;
|
|
char* stack_copy;
|
|
intptr_t _stack_saved;
|
|
StackState* stack_prev;
|
|
inline int copy_stack_to_heap_up_to(const char* const stop) G_NOEXCEPT;
|
|
inline void free_stack_copy() G_NOEXCEPT;
|
|
|
|
public:
|
|
/**
|
|
* Creates a started, but inactive, state, using *current*
|
|
* as the previous.
|
|
*/
|
|
StackState(void* mark, StackState& current);
|
|
/**
|
|
* Creates an inactive, unstarted, state.
|
|
*/
|
|
StackState();
|
|
~StackState();
|
|
StackState(const StackState& other);
|
|
StackState& operator=(const StackState& other);
|
|
inline void copy_heap_to_stack(const StackState& current) G_NOEXCEPT;
|
|
inline int copy_stack_to_heap(char* const stackref, const StackState& current) G_NOEXCEPT;
|
|
inline bool started() const G_NOEXCEPT;
|
|
inline bool main() const G_NOEXCEPT;
|
|
inline bool active() const G_NOEXCEPT;
|
|
inline void set_active() G_NOEXCEPT;
|
|
inline void set_inactive() G_NOEXCEPT;
|
|
inline intptr_t stack_saved() const G_NOEXCEPT;
|
|
inline char* stack_start() const G_NOEXCEPT;
|
|
static inline StackState make_main() G_NOEXCEPT;
|
|
#ifdef GREENLET_USE_STDIO
|
|
friend std::ostream& operator<<(std::ostream& os, const StackState& s);
|
|
#endif
|
|
};
|
|
#ifdef GREENLET_USE_STDIO
|
|
std::ostream& operator<<(std::ostream& os, const StackState& s);
|
|
#endif
|
|
|
|
class SwitchingArgs
|
|
{
|
|
private:
|
|
G_NO_ASSIGNMENT_OF_CLS(SwitchingArgs);
|
|
// If args and kwargs are both false (NULL), this is a *throw*, not a
|
|
// switch. PyErr_... must have been called already.
|
|
OwnedObject _args;
|
|
OwnedObject _kwargs;
|
|
public:
|
|
|
|
SwitchingArgs()
|
|
{}
|
|
|
|
SwitchingArgs(const OwnedObject& args, const OwnedObject& kwargs)
|
|
: _args(args),
|
|
_kwargs(kwargs)
|
|
{}
|
|
|
|
SwitchingArgs(const SwitchingArgs& other)
|
|
: _args(other._args),
|
|
_kwargs(other._kwargs)
|
|
{}
|
|
|
|
OwnedObject& args()
|
|
{
|
|
return this->_args;
|
|
}
|
|
|
|
OwnedObject& kwargs()
|
|
{
|
|
return this->_kwargs;
|
|
}
|
|
|
|
/**
|
|
* Moves ownership from the argument to this object.
|
|
*/
|
|
SwitchingArgs& operator<<=(SwitchingArgs& other)
|
|
{
|
|
if (this != &other) {
|
|
this->_args = other._args;
|
|
this->_kwargs = other._kwargs;
|
|
other.CLEAR();
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
/**
|
|
* Acquires ownership of the argument (consumes the reference).
|
|
*/
|
|
SwitchingArgs& operator<<=(PyObject* args)
|
|
{
|
|
this->_args = OwnedObject::consuming(args);
|
|
this->_kwargs.CLEAR();
|
|
return *this;
|
|
}
|
|
|
|
/**
|
|
* Acquires ownership of the argument.
|
|
*
|
|
* Sets the args to be the given value; clears the kwargs.
|
|
*/
|
|
SwitchingArgs& operator<<=(OwnedObject& args)
|
|
{
|
|
assert(&args != &this->_args);
|
|
this->_args = args;
|
|
this->_kwargs.CLEAR();
|
|
args.CLEAR();
|
|
|
|
return *this;
|
|
}
|
|
|
|
G_EXPLICIT_OP operator bool() const G_NOEXCEPT
|
|
{
|
|
return this->_args || this->_kwargs;
|
|
}
|
|
|
|
inline void CLEAR()
|
|
{
|
|
this->_args.CLEAR();
|
|
this->_kwargs.CLEAR();
|
|
}
|
|
};
|
|
|
|
class ThreadState;
|
|
|
|
class UserGreenlet;
|
|
class MainGreenlet;
|
|
|
|
class Greenlet
|
|
{
|
|
private:
|
|
G_NO_COPIES_OF_CLS(Greenlet);
|
|
private:
|
|
// XXX: Work to remove these.
|
|
friend class ThreadState;
|
|
friend class UserGreenlet;
|
|
friend class MainGreenlet;
|
|
protected:
|
|
ExceptionState exception_state;
|
|
SwitchingArgs switch_args;
|
|
StackState stack_state;
|
|
PythonState python_state;
|
|
Greenlet(PyGreenlet* p, const StackState& initial_state);
|
|
public:
|
|
Greenlet(PyGreenlet* p);
|
|
virtual ~Greenlet();
|
|
|
|
template <typename IsPy37> // maybe we can use a value here?
|
|
const OwnedObject context(const typename IsPy37::IsIt=nullptr) const;
|
|
|
|
template <typename IsPy37>
|
|
inline void context(refs::BorrowedObject new_context, typename IsPy37::IsIt=nullptr);
|
|
|
|
inline SwitchingArgs& args()
|
|
{
|
|
return this->switch_args;
|
|
}
|
|
|
|
virtual const refs::BorrowedMainGreenlet main_greenlet() const = 0;
|
|
|
|
inline intptr_t stack_saved() const G_NOEXCEPT
|
|
{
|
|
return this->stack_state.stack_saved();
|
|
}
|
|
|
|
// This is used by the macro SLP_SAVE_STATE to compute the
|
|
// difference in stack sizes. It might be nice to handle the
|
|
// computation ourself, but the type of the result
|
|
// varies by platform, so doing it in the macro is the
|
|
// simplest way.
|
|
inline const char* stack_start() const G_NOEXCEPT
|
|
{
|
|
return this->stack_state.stack_start();
|
|
}
|
|
|
|
virtual OwnedObject throw_GreenletExit_during_dealloc(const ThreadState& current_thread_state);
|
|
virtual OwnedObject g_switch() = 0;
|
|
/**
|
|
* Force the greenlet to appear dead. Used when it's not
|
|
* possible to throw an exception into a greenlet anymore.
|
|
*
|
|
* This losses access to the thread state and the main greenlet.
|
|
*/
|
|
virtual void murder_in_place();
|
|
|
|
/**
|
|
* Called when somebody notices we were running in a dead
|
|
* thread to allow cleaning up resources (because we can't
|
|
* raise GreenletExit into it anymore).
|
|
* This is very similar to ``murder_in_place()``, except that
|
|
* it DOES NOT lose the main greenlet or thread state.
|
|
*/
|
|
inline void deactivate_and_free();
|
|
|
|
|
|
// Called when some thread wants to deallocate a greenlet
|
|
// object.
|
|
// The thread may or may not be the same thread the greenlet
|
|
// was running in.
|
|
// The thread state will be null if the thread the greenlet
|
|
// was running in was known to have exited.
|
|
void deallocing_greenlet_in_thread(const ThreadState* current_state);
|
|
|
|
// TODO: Figure out how to make these non-public.
|
|
inline void slp_restore_state() G_NOEXCEPT;
|
|
inline int slp_save_state(char *const stackref) G_NOEXCEPT;
|
|
|
|
inline bool is_currently_running_in_some_thread() const;
|
|
virtual bool belongs_to_thread(const ThreadState* state) const;
|
|
|
|
inline bool started() const
|
|
{
|
|
return this->stack_state.started();
|
|
}
|
|
inline bool active() const
|
|
{
|
|
return this->stack_state.active();
|
|
}
|
|
inline bool main() const
|
|
{
|
|
return this->stack_state.main();
|
|
}
|
|
virtual refs::BorrowedMainGreenlet find_main_greenlet_in_lineage() const = 0;
|
|
|
|
virtual const OwnedGreenlet parent() const = 0;
|
|
virtual void parent(const refs::BorrowedObject new_parent) = 0;
|
|
|
|
inline const PythonState::OwnedFrame& top_frame()
|
|
{
|
|
return this->python_state.top_frame();
|
|
}
|
|
|
|
virtual const OwnedObject& run() const = 0;
|
|
virtual void run(const refs::BorrowedObject nrun) = 0;
|
|
|
|
|
|
virtual int tp_traverse(visitproc visit, void* arg);
|
|
virtual int tp_clear();
|
|
|
|
|
|
// Return the thread state that the greenlet is running in, or
|
|
// null if the greenlet is not running or the thread is known
|
|
// to have exited.
|
|
virtual ThreadState* thread_state() const G_NOEXCEPT = 0;
|
|
|
|
// Return true if the greenlet is known to have been running
|
|
// (active) in a thread that has now exited.
|
|
virtual bool was_running_in_dead_thread() const G_NOEXCEPT = 0;
|
|
|
|
// Return a borrowed greenlet that is the Python object
|
|
// this object represents.
|
|
virtual BorrowedGreenlet self() const G_NOEXCEPT = 0;
|
|
|
|
protected:
|
|
inline void release_args();
|
|
|
|
// The functions that must not be inlined are declared virtual.
|
|
// We also mark them as protected, not private, so that the
|
|
// compiler is forced to call them through a function pointer.
|
|
// (A sufficiently smart compiler could directly call a private
|
|
// virtual function since it can never be overridden in a
|
|
// subclass).
|
|
|
|
// Also TODO: Switch away from integer error codes and to enums,
|
|
// or throw exceptions when possible.
|
|
struct switchstack_result_t
|
|
{
|
|
int status;
|
|
Greenlet* the_state_that_switched;
|
|
OwnedGreenlet origin_greenlet;
|
|
|
|
switchstack_result_t()
|
|
: status(0),
|
|
the_state_that_switched(nullptr)
|
|
{}
|
|
|
|
switchstack_result_t(int err)
|
|
: status(err),
|
|
the_state_that_switched(nullptr)
|
|
{}
|
|
|
|
switchstack_result_t(int err, Greenlet* state, OwnedGreenlet& origin)
|
|
: status(err),
|
|
the_state_that_switched(state),
|
|
origin_greenlet(origin)
|
|
{
|
|
}
|
|
|
|
switchstack_result_t(int err, Greenlet* state, const BorrowedGreenlet& origin)
|
|
: status(err),
|
|
the_state_that_switched(state),
|
|
origin_greenlet(origin)
|
|
{
|
|
}
|
|
|
|
switchstack_result_t& operator=(const switchstack_result_t& other)
|
|
{
|
|
this->status = other.status;
|
|
this->the_state_that_switched = other.the_state_that_switched;
|
|
this->origin_greenlet = other.origin_greenlet;
|
|
return *this;
|
|
}
|
|
};
|
|
|
|
// Returns the previous greenlet we just switched away from.
|
|
virtual OwnedGreenlet g_switchstack_success() G_NOEXCEPT;
|
|
|
|
|
|
// Check the preconditions for switching to this greenlet; if they
|
|
// aren't met, throws PyErrOccurred. Most callers will want to
|
|
// catch this and clear the arguments
|
|
inline void check_switch_allowed() const;
|
|
class GreenletStartedWhileInPython : public std::runtime_error
|
|
{
|
|
public:
|
|
GreenletStartedWhileInPython() : std::runtime_error("")
|
|
{}
|
|
};
|
|
|
|
protected:
|
|
|
|
|
|
/**
|
|
Perform a stack switch into this greenlet.
|
|
|
|
This temporarily sets the global variable
|
|
``switching_thread_state`` to this greenlet; as soon as the
|
|
call to ``slp_switch`` completes, this is reset to NULL.
|
|
Consequently, this depends on the GIL.
|
|
|
|
TODO: Adopt the stackman model and pass ``slp_switch`` a
|
|
callback function and context pointer; this eliminates the
|
|
need for global variables altogether.
|
|
|
|
Because the stack switch happens in this function, this
|
|
function can't use its own stack (local) variables, set
|
|
before the switch, and then accessed after the switch.
|
|
|
|
Further, you con't even access ``g_thread_state_global``
|
|
before and after the switch from the global variable.
|
|
Because it is thread local some compilers cache it in a
|
|
register/on the stack, notably new versions of MSVC; this
|
|
breaks with strange crashes sometime later, because writing
|
|
to anything in ``g_thread_state_global`` after the switch
|
|
is actually writing to random memory. For this reason, we
|
|
call a non-inlined function to finish the operation. (XXX:
|
|
The ``/GT`` MSVC compiler argument probably fixes that.)
|
|
|
|
It is very important that stack switch is 'atomic', i.e. no
|
|
calls into other Python code allowed (except very few that
|
|
are safe), because global variables are very fragile. (This
|
|
should no longer be the case with thread-local variables.)
|
|
|
|
*/
|
|
switchstack_result_t g_switchstack(void);
|
|
private:
|
|
OwnedObject g_switch_finish(const switchstack_result_t& err);
|
|
|
|
};
|
|
|
|
class UserGreenlet : public Greenlet
|
|
{
|
|
private:
|
|
static greenlet::PythonAllocator<UserGreenlet> allocator;
|
|
BorrowedGreenlet _self;
|
|
OwnedMainGreenlet _main_greenlet;
|
|
OwnedObject _run_callable;
|
|
OwnedGreenlet _parent;
|
|
public:
|
|
static void* operator new(size_t UNUSED(count));
|
|
static void operator delete(void* ptr);
|
|
|
|
UserGreenlet(PyGreenlet* p, BorrowedGreenlet the_parent);
|
|
virtual ~UserGreenlet();
|
|
|
|
virtual refs::BorrowedMainGreenlet find_main_greenlet_in_lineage() const;
|
|
virtual bool was_running_in_dead_thread() const G_NOEXCEPT;
|
|
virtual ThreadState* thread_state() const G_NOEXCEPT;
|
|
virtual OwnedObject g_switch();
|
|
virtual const OwnedObject& run() const
|
|
{
|
|
if (this->started() || !this->_run_callable) {
|
|
throw AttributeError("run");
|
|
}
|
|
return this->_run_callable;
|
|
}
|
|
virtual void run(const refs::BorrowedObject nrun);
|
|
|
|
virtual const OwnedGreenlet parent() const;
|
|
virtual void parent(const refs::BorrowedObject new_parent);
|
|
|
|
virtual const refs::BorrowedMainGreenlet main_greenlet() const;
|
|
|
|
virtual BorrowedGreenlet self() const G_NOEXCEPT;
|
|
virtual void murder_in_place();
|
|
virtual bool belongs_to_thread(const ThreadState* state) const;
|
|
virtual int tp_traverse(visitproc visit, void* arg);
|
|
virtual int tp_clear();
|
|
class ParentIsCurrentGuard
|
|
{
|
|
private:
|
|
OwnedGreenlet oldparent;
|
|
UserGreenlet* greenlet;
|
|
G_NO_COPIES_OF_CLS(ParentIsCurrentGuard);
|
|
public:
|
|
ParentIsCurrentGuard(UserGreenlet* p, const ThreadState& thread_state);
|
|
~ParentIsCurrentGuard();
|
|
};
|
|
virtual OwnedObject throw_GreenletExit_during_dealloc(const ThreadState& current_thread_state);
|
|
protected:
|
|
virtual switchstack_result_t g_initialstub(void* mark);
|
|
private:
|
|
void inner_bootstrap(OwnedGreenlet& origin_greenlet, OwnedObject& run) G_NOEXCEPT_WIN32;
|
|
};
|
|
|
|
class MainGreenlet : public Greenlet
|
|
{
|
|
private:
|
|
static greenlet::PythonAllocator<MainGreenlet> allocator;
|
|
refs::BorrowedMainGreenlet _self;
|
|
ThreadState* _thread_state;
|
|
G_NO_COPIES_OF_CLS(MainGreenlet);
|
|
public:
|
|
static void* operator new(size_t UNUSED(count));
|
|
static void operator delete(void* ptr);
|
|
|
|
MainGreenlet(refs::BorrowedMainGreenlet::PyType*, ThreadState*);
|
|
virtual ~MainGreenlet();
|
|
|
|
|
|
virtual const OwnedObject& run() const;
|
|
virtual void run(const refs::BorrowedObject nrun);
|
|
|
|
virtual const OwnedGreenlet parent() const;
|
|
virtual void parent(const refs::BorrowedObject new_parent);
|
|
|
|
virtual const refs::BorrowedMainGreenlet main_greenlet() const;
|
|
|
|
virtual refs::BorrowedMainGreenlet find_main_greenlet_in_lineage() const;
|
|
virtual bool was_running_in_dead_thread() const G_NOEXCEPT;
|
|
virtual ThreadState* thread_state() const G_NOEXCEPT;
|
|
void thread_state(ThreadState*) G_NOEXCEPT;
|
|
virtual OwnedObject g_switch();
|
|
virtual BorrowedGreenlet self() const G_NOEXCEPT;
|
|
virtual int tp_traverse(visitproc visit, void* arg);
|
|
};
|
|
|
|
};
|
|
|
|
template<typename T>
|
|
void greenlet::operator<<(const PyThreadState *const lhs, T& rhs)
|
|
{
|
|
rhs.operator<<(lhs);
|
|
}
|
|
|
|
using greenlet::ExceptionState;
|
|
|
|
ExceptionState::ExceptionState()
|
|
{
|
|
this->clear();
|
|
}
|
|
|
|
#if PY_VERSION_HEX >= 0x030700A3
|
|
// ******** Python 3.7 and above *********
|
|
void ExceptionState::operator<<(const PyThreadState *const tstate) G_NOEXCEPT
|
|
{
|
|
this->exc_info = tstate->exc_info;
|
|
this->exc_state = tstate->exc_state;
|
|
}
|
|
|
|
void ExceptionState::operator>>(PyThreadState *const tstate) G_NOEXCEPT
|
|
{
|
|
tstate->exc_state = this->exc_state;
|
|
tstate->exc_info =
|
|
this->exc_info ? this->exc_info : &tstate->exc_state;
|
|
this->clear();
|
|
}
|
|
|
|
void ExceptionState::clear() G_NOEXCEPT
|
|
{
|
|
this->exc_info = nullptr;
|
|
this->exc_state.exc_value = nullptr;
|
|
#if !GREENLET_PY311
|
|
this->exc_state.exc_type = nullptr;
|
|
this->exc_state.exc_traceback = nullptr;
|
|
#endif
|
|
this->exc_state.previous_item = nullptr;
|
|
}
|
|
|
|
int ExceptionState::tp_traverse(visitproc visit, void* arg) G_NOEXCEPT
|
|
{
|
|
Py_VISIT(this->exc_state.exc_value);
|
|
#if !GREENLET_PY311
|
|
Py_VISIT(this->exc_state.exc_type);
|
|
Py_VISIT(this->exc_state.exc_traceback);
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
void ExceptionState::tp_clear() G_NOEXCEPT
|
|
{
|
|
Py_CLEAR(this->exc_state.exc_value);
|
|
#if !GREENLET_PY311
|
|
Py_CLEAR(this->exc_state.exc_type);
|
|
Py_CLEAR(this->exc_state.exc_traceback);
|
|
#endif
|
|
}
|
|
#else
|
|
// ********** Python 3.6 and below ********
|
|
void ExceptionState::operator<<(const PyThreadState *const tstate) G_NOEXCEPT
|
|
{
|
|
this->exc_value.steal(tstate->exc_value);
|
|
#if !GREENLET_PY311
|
|
this->exc_type.steal(tstate->exc_type);
|
|
this->exc_traceback.steal(tstate->exc_traceback);
|
|
#endif
|
|
}
|
|
|
|
void ExceptionState::operator>>(PyThreadState *const tstate) G_NOEXCEPT
|
|
{
|
|
tstate->exc_value <<= this->exc_value;
|
|
#if !GREENLET_PY311
|
|
tstate->exc_type <<= this->exc_type;
|
|
tstate->exc_traceback <<= this->exc_traceback;
|
|
#endif
|
|
this->clear();
|
|
}
|
|
|
|
void ExceptionState::clear() G_NOEXCEPT
|
|
{
|
|
this->exc_value = nullptr;
|
|
#if !GREENLET_PY311
|
|
this->exc_type = nullptr;
|
|
this->exc_traceback = nullptr;
|
|
#endif
|
|
}
|
|
|
|
int ExceptionState::tp_traverse(visitproc visit, void* arg) G_NOEXCEPT
|
|
{
|
|
Py_VISIT(this->exc_value.borrow());
|
|
#if !GREENLET_PY311
|
|
Py_VISIT(this->exc_type.borrow());
|
|
Py_VISIT(this->exc_traceback.borrow());
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
void ExceptionState::tp_clear() G_NOEXCEPT
|
|
{
|
|
this->exc_value.CLEAR();
|
|
#if !GREENLET_PY311
|
|
this->exc_type.CLEAR();
|
|
this->exc_traceback.CLEAR();
|
|
#endif
|
|
}
|
|
#endif
|
|
|
|
|
|
using greenlet::PythonState;
|
|
|
|
PythonState::PythonState()
|
|
: _top_frame()
|
|
#if GREENLET_USE_CFRAME
|
|
,cframe(nullptr)
|
|
,use_tracing(0)
|
|
#endif
|
|
,recursion_depth(0)
|
|
,trash_delete_nesting(0)
|
|
#if GREENLET_PY311
|
|
,current_frame(nullptr)
|
|
,datastack_chunk(nullptr)
|
|
,datastack_top(nullptr)
|
|
,datastack_limit(nullptr)
|
|
#endif
|
|
{
|
|
#if GREENLET_USE_CFRAME
|
|
/*
|
|
The PyThreadState->cframe pointer usually points to memory on
|
|
the stack, alloceted in a call into PyEval_EvalFrameDefault.
|
|
|
|
Initially, before any evaluation begins, it points to the
|
|
initial PyThreadState object's ``root_cframe`` object, which is
|
|
statically allocated for the lifetime of the thread.
|
|
|
|
A greenlet can last for longer than a call to
|
|
PyEval_EvalFrameDefault, so we can't set its ``cframe`` pointer
|
|
to be the current ``PyThreadState->cframe``; nor could we use
|
|
one from the greenlet parent for the same reason. Yet a further
|
|
no: we can't allocate one scoped to the greenlet and then
|
|
destroy it when the greenlet is deallocated, because inside the
|
|
interpreter the _PyCFrame objects form a linked list, and that too
|
|
can result in accessing memory beyond its dynamic lifetime (if
|
|
the greenlet doesn't actually finish before it dies, its entry
|
|
could still be in the list).
|
|
|
|
Using the ``root_cframe`` is problematic, though, because its
|
|
members are never modified by the interpreter and are set to 0,
|
|
meaning that its ``use_tracing`` flag is never updated. We don't
|
|
want to modify that value in the ``root_cframe`` ourself: it
|
|
*shouldn't* matter much because we should probably never get
|
|
back to the point where that's the only cframe on the stack;
|
|
even if it did matter, the major consequence of an incorrect
|
|
value for ``use_tracing`` is that if its true the interpreter
|
|
does some extra work --- however, it's just good code hygiene.
|
|
|
|
Our solution: before a greenlet runs, after its initial
|
|
creation, it uses the ``root_cframe`` just to have something to
|
|
put there. However, once the greenlet is actually switched to
|
|
for the first time, ``g_initialstub`` (which doesn't actually
|
|
"return" while the greenlet is running) stores a new _PyCFrame on
|
|
its local stack, and copies the appropriate values from the
|
|
currently running _PyCFrame; this is then made the _PyCFrame for the
|
|
newly-minted greenlet. ``g_initialstub`` then proceeds to call
|
|
``glet.run()``, which results in ``PyEval_...`` adding the
|
|
_PyCFrame to the list. Switches continue as normal. Finally, when
|
|
the greenlet finishes, the call to ``glet.run()`` returns and
|
|
the _PyCFrame is taken out of the linked list and the stack value
|
|
is now unused and free to expire.
|
|
|
|
XXX: I think we can do better. If we're deallocing in the same
|
|
thread, can't we traverse the list and unlink our frame?
|
|
Can we just keep a reference to the thread state in case we
|
|
dealloc in another thread? (Is that even possible if we're still
|
|
running and haven't returned from g_initialstub?)
|
|
*/
|
|
this->cframe = &PyThreadState_GET()->root_cframe;
|
|
#endif
|
|
}
|
|
|
|
void PythonState::operator<<(const PyThreadState *const tstate) G_NOEXCEPT
|
|
{
|
|
#if GREENLET_PY37
|
|
this->_context.steal(tstate->context);
|
|
#endif
|
|
#if GREENLET_USE_CFRAME
|
|
/*
|
|
IMPORTANT: ``cframe`` is a pointer into the STACK. Thus, because
|
|
the call to ``slp_switch()`` changes the contents of the stack,
|
|
you cannot read from ``ts_current->cframe`` after that call and
|
|
necessarily get the same values you get from reading it here.
|
|
Anything you need to restore from now to then must be saved in a
|
|
global/threadlocal variable (because we can't use stack
|
|
variables here either). For things that need to persist across
|
|
the switch, use `will_switch_from`.
|
|
*/
|
|
this->cframe = tstate->cframe;
|
|
this->use_tracing = tstate->cframe->use_tracing;
|
|
#endif
|
|
#if GREENLET_PY311
|
|
this->recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
|
|
this->current_frame = tstate->cframe->current_frame;
|
|
this->datastack_chunk = tstate->datastack_chunk;
|
|
this->datastack_top = tstate->datastack_top;
|
|
this->datastack_limit = tstate->datastack_limit;
|
|
PyFrameObject *frame = PyThreadState_GetFrame((PyThreadState *)tstate);
|
|
Py_XDECREF(frame); // PyThreadState_GetFrame gives us a new reference.
|
|
this->_top_frame.steal(frame);
|
|
#else
|
|
this->recursion_depth = tstate->recursion_depth;
|
|
this->_top_frame.steal(tstate->frame);
|
|
#endif
|
|
|
|
// All versions of Python.
|
|
this->trash_delete_nesting = tstate->trash_delete_nesting;
|
|
}
|
|
|
|
void PythonState::operator>>(PyThreadState *const tstate) G_NOEXCEPT
|
|
{
|
|
#if GREENLET_PY37
|
|
tstate->context = this->_context.relinquish_ownership();
|
|
/* Incrementing this value invalidates the contextvars cache,
|
|
which would otherwise remain valid across switches */
|
|
tstate->context_ver++;
|
|
#endif
|
|
#if GREENLET_USE_CFRAME
|
|
tstate->cframe = this->cframe;
|
|
/*
|
|
If we were tracing, we need to keep tracing.
|
|
There should never be the possibility of hitting the
|
|
root_cframe here. See note above about why we can't
|
|
just copy this from ``origin->cframe->use_tracing``.
|
|
*/
|
|
tstate->cframe->use_tracing = this->use_tracing;
|
|
#endif
|
|
#if GREENLET_PY311
|
|
tstate->recursion_remaining = tstate->recursion_limit - this->recursion_depth;
|
|
tstate->cframe->current_frame = this->current_frame;
|
|
tstate->datastack_chunk = this->datastack_chunk;
|
|
tstate->datastack_top = this->datastack_top;
|
|
tstate->datastack_limit = this->datastack_limit;
|
|
this->_top_frame.relinquish_ownership();
|
|
#else
|
|
tstate->frame = this->_top_frame.relinquish_ownership();
|
|
tstate->recursion_depth = this->recursion_depth;
|
|
#endif
|
|
// All versions of Python.
|
|
tstate->trash_delete_nesting = this->trash_delete_nesting;
|
|
}
|
|
|
|
void PythonState::will_switch_from(PyThreadState *const origin_tstate) G_NOEXCEPT
|
|
{
|
|
#if GREENLET_USE_CFRAME
|
|
// The weird thing is, we don't actually save this for an
|
|
// effect on the current greenlet, it's saved for an
|
|
// effect on the target greenlet. That is, we want
|
|
// continuity of this setting across the greenlet switch.
|
|
this->use_tracing = origin_tstate->cframe->use_tracing;
|
|
#endif
|
|
}
|
|
|
|
void PythonState::set_initial_state(const PyThreadState* const tstate) G_NOEXCEPT
|
|
{
|
|
this->_top_frame = nullptr;
|
|
#if GREENLET_PY311
|
|
this->recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
|
|
#else
|
|
this->recursion_depth = tstate->recursion_depth;
|
|
#endif
|
|
}
|
|
// TODO: Better state management about when we own the top frame.
|
|
int PythonState::tp_traverse(visitproc visit, void* arg, bool own_top_frame) G_NOEXCEPT
|
|
{
|
|
#if GREENLET_PY37
|
|
Py_VISIT(this->_context.borrow());
|
|
#endif
|
|
if (own_top_frame) {
|
|
Py_VISIT(this->_top_frame.borrow());
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void PythonState::tp_clear(bool own_top_frame) G_NOEXCEPT
|
|
{
|
|
PythonStateContext::tp_clear();
|
|
// If we get here owning a frame,
|
|
// we got dealloc'd without being finished. We may or may not be
|
|
// in the same thread.
|
|
if (own_top_frame) {
|
|
this->_top_frame.CLEAR();
|
|
}
|
|
}
|
|
|
|
#if GREENLET_USE_CFRAME
|
|
void PythonState::set_new_cframe(_PyCFrame& frame) G_NOEXCEPT
|
|
{
|
|
frame = *PyThreadState_GET()->cframe;
|
|
/* Make the target greenlet refer to the stack value. */
|
|
this->cframe = &frame;
|
|
/*
|
|
And restore the link to the previous frame so this one gets
|
|
unliked appropriately.
|
|
*/
|
|
this->cframe->previous = &PyThreadState_GET()->root_cframe;
|
|
}
|
|
#endif
|
|
|
|
const PythonState::OwnedFrame& PythonState::top_frame() const G_NOEXCEPT
|
|
{
|
|
return this->_top_frame;
|
|
}
|
|
|
|
void PythonState::did_finish(PyThreadState* tstate) G_NOEXCEPT
|
|
{
|
|
#if GREENLET_PY311
|
|
// See https://github.com/gevent/gevent/issues/1924 and
|
|
// https://github.com/python-greenlet/greenlet/issues/328. In
|
|
// short, Python 3.11 allocates memory for frames as a sort of
|
|
// linked list that's kept as part of PyThreadState in the
|
|
// ``datastack_chunk`` member and friends. These are saved and
|
|
// restored as part of switching greenlets.
|
|
//
|
|
// When we initially switch to a greenlet, we set those to NULL.
|
|
// That causes the frame management code to treat this like a
|
|
// brand new thread and start a fresh list of chunks, beginning
|
|
// with a new "root" chunk. As we make calls in this greenlet,
|
|
// those chunks get added, and as calls return, they get popped.
|
|
// But the frame code (pystate.c) is careful to make sure that the
|
|
// root chunk never gets popped.
|
|
//
|
|
// Thus, when a greenlet exits for the last time, there will be at
|
|
// least a single root chunk that we must be responsible for
|
|
// deallocating.
|
|
//
|
|
// The complex part is that these chunks are allocated and freed
|
|
// using ``_PyObject_VirtualAlloc``/``Free``. Those aren't public
|
|
// functions, and they aren't exported for linking. It so happens
|
|
// that we know they are just thin wrappers around the Arena
|
|
// allocator, so we can use that directly to deallocate in a
|
|
// compatible way.
|
|
//
|
|
// CAUTION: Check this implementation detail on every major version.
|
|
//
|
|
// It might be nice to be able to do this in our destructor, but
|
|
// can we be sure that no one else is using that memory? Plus, as
|
|
// described below, our pointers may not even be valid anymore. As
|
|
// a special case, there is one time that we know we can do this,
|
|
// and that's from the destructor of the associated UserGreenlet
|
|
// (NOT main greenlet)
|
|
PyObjectArenaAllocator alloc;
|
|
_PyStackChunk* chunk = nullptr;
|
|
if (tstate) {
|
|
// We really did finish, we can never be switched to again.
|
|
chunk = tstate->datastack_chunk;
|
|
// Unfortunately, we can't do much sanity checking. Our
|
|
// this->datastack_chunk pointer is out of date (evaluation may
|
|
// have popped down through it already) so we can't verify that
|
|
// we deallocate it. I don't think we can even check datastack_top
|
|
// for the same reason.
|
|
|
|
PyObject_GetArenaAllocator(&alloc);
|
|
tstate->datastack_chunk = nullptr;
|
|
tstate->datastack_limit = nullptr;
|
|
tstate->datastack_top = nullptr;
|
|
|
|
}
|
|
else if (this->datastack_chunk) {
|
|
// The UserGreenlet (NOT the main greenlet!) is being deallocated. If we're
|
|
// still holding a stack chunk, it's garbage because we know
|
|
// we can never switch back to let cPython clean it up.
|
|
// Because the last time we got switched away from, and we
|
|
// haven't run since then, we know our chain is valid and can
|
|
// be dealloced.
|
|
chunk = this->datastack_chunk;
|
|
PyObject_GetArenaAllocator(&alloc);
|
|
}
|
|
|
|
if (alloc.free && chunk) {
|
|
// In case the arena mechanism has been torn down already.
|
|
while (chunk) {
|
|
_PyStackChunk *prev = chunk->previous;
|
|
chunk->previous = nullptr;
|
|
alloc.free(alloc.ctx, chunk, chunk->size);
|
|
chunk = prev;
|
|
}
|
|
}
|
|
|
|
this->datastack_chunk = nullptr;
|
|
this->datastack_limit = nullptr;
|
|
this->datastack_top = nullptr;
|
|
#endif
|
|
}
|
|
|
|
|
|
|
|
|
|
using greenlet::StackState;
|
|
|
|
#ifdef GREENLET_USE_STDIO
|
|
#include <iostream>
|
|
using std::cerr;
|
|
using std::endl;
|
|
|
|
std::ostream& greenlet::operator<<(std::ostream& os, const StackState& s)
|
|
{
|
|
os << "StackState(stack_start=" << (void*)s._stack_start
|
|
<< ", stack_stop=" << (void*)s.stack_stop
|
|
<< ", stack_copy=" << (void*)s.stack_copy
|
|
<< ", stack_saved=" << s._stack_saved
|
|
<< ", stack_prev=" << s.stack_prev
|
|
<< ", addr=" << &s
|
|
<< ")";
|
|
return os;
|
|
}
|
|
#endif
|
|
|
|
StackState::StackState(void* mark, StackState& current)
|
|
: _stack_start(nullptr),
|
|
stack_stop((char*)mark),
|
|
stack_copy(nullptr),
|
|
_stack_saved(0),
|
|
/* Skip a dying greenlet */
|
|
stack_prev(current._stack_start
|
|
? ¤t
|
|
: current.stack_prev)
|
|
{
|
|
}
|
|
|
|
StackState::StackState()
|
|
: _stack_start(nullptr),
|
|
stack_stop(nullptr),
|
|
stack_copy(nullptr),
|
|
_stack_saved(0),
|
|
stack_prev(nullptr)
|
|
{
|
|
}
|
|
|
|
StackState::StackState(const StackState& other)
|
|
// can't use a delegating constructor because of
|
|
// MSVC for Python 2.7
|
|
: _stack_start(nullptr),
|
|
stack_stop(nullptr),
|
|
stack_copy(nullptr),
|
|
_stack_saved(0),
|
|
stack_prev(nullptr)
|
|
{
|
|
this->operator=(other);
|
|
}
|
|
|
|
StackState& StackState::operator=(const StackState& other)
|
|
{
|
|
if (&other == this) {
|
|
return *this;
|
|
}
|
|
if (other._stack_saved) {
|
|
throw std::runtime_error("Refusing to steal memory.");
|
|
}
|
|
|
|
//If we have memory allocated, dispose of it
|
|
this->free_stack_copy();
|
|
|
|
this->_stack_start = other._stack_start;
|
|
this->stack_stop = other.stack_stop;
|
|
this->stack_copy = other.stack_copy;
|
|
this->_stack_saved = other._stack_saved;
|
|
this->stack_prev = other.stack_prev;
|
|
return *this;
|
|
}
|
|
|
|
inline void StackState::free_stack_copy() G_NOEXCEPT
|
|
{
|
|
PyMem_Free(this->stack_copy);
|
|
this->stack_copy = nullptr;
|
|
this->_stack_saved = 0;
|
|
}
|
|
|
|
inline void StackState::copy_heap_to_stack(const StackState& current) G_NOEXCEPT
|
|
{
|
|
// cerr << "copy_heap_to_stack" << endl
|
|
// << "\tFrom : " << *this << endl
|
|
// << "\tCurrent:" << current
|
|
// << endl;
|
|
/* Restore the heap copy back into the C stack */
|
|
if (this->_stack_saved != 0) {
|
|
memcpy(this->_stack_start, this->stack_copy, this->_stack_saved);
|
|
this->free_stack_copy();
|
|
}
|
|
StackState* owner = const_cast<StackState*>(¤t);
|
|
if (!owner->_stack_start) {
|
|
owner = owner->stack_prev; /* greenlet is dying, skip it */
|
|
}
|
|
while (owner && owner->stack_stop <= this->stack_stop) {
|
|
// cerr << "\tOwner: " << owner << endl;
|
|
owner = owner->stack_prev; /* find greenlet with more stack */
|
|
}
|
|
this->stack_prev = owner;
|
|
// cerr << "\tFinished with: " << *this << endl;
|
|
}
|
|
|
|
inline int StackState::copy_stack_to_heap_up_to(const char* const stop) G_NOEXCEPT
|
|
{
|
|
/* Save more of g's stack into the heap -- at least up to 'stop'
|
|
g->stack_stop |________|
|
|
| |
|
|
| __ stop . . . . .
|
|
| | ==> . .
|
|
|________| _______
|
|
| | | |
|
|
| | | |
|
|
g->stack_start | | |_______| g->stack_copy
|
|
*/
|
|
intptr_t sz1 = this->_stack_saved;
|
|
intptr_t sz2 = stop - this->_stack_start;
|
|
assert(this->_stack_start);
|
|
if (sz2 > sz1) {
|
|
char* c = (char*)PyMem_Realloc(this->stack_copy, sz2);
|
|
if (!c) {
|
|
PyErr_NoMemory();
|
|
return -1;
|
|
}
|
|
memcpy(c + sz1, this->_stack_start + sz1, sz2 - sz1);
|
|
this->stack_copy = c;
|
|
this->_stack_saved = sz2;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
inline int StackState::copy_stack_to_heap(char* const stackref,
|
|
const StackState& current) G_NOEXCEPT
|
|
{
|
|
// cerr << "copy_stack_to_heap: " << endl
|
|
// << "\tstackref: " << (void*)stackref << endl
|
|
// << "\tthis: " << *this << endl
|
|
// << "\tcurrent: " << current
|
|
// << endl;
|
|
/* must free all the C stack up to target_stop */
|
|
const char* const target_stop = this->stack_stop;
|
|
|
|
StackState* owner = const_cast<StackState*>(¤t);
|
|
assert(owner->_stack_saved == 0); // everything is present on the stack
|
|
if (!owner->_stack_start) {
|
|
// cerr << "\tcurrent is dead; using: " << owner->stack_prev << endl;
|
|
owner = owner->stack_prev; /* not saved if dying */
|
|
}
|
|
else {
|
|
owner->_stack_start = stackref;
|
|
}
|
|
|
|
while (owner->stack_stop < target_stop) {
|
|
// cerr << "\tCopying from " << *owner << endl;
|
|
/* ts_current is entierely within the area to free */
|
|
if (owner->copy_stack_to_heap_up_to(owner->stack_stop)) {
|
|
return -1; /* XXX */
|
|
}
|
|
owner = owner->stack_prev;
|
|
}
|
|
if (owner != this) {
|
|
if (owner->copy_stack_to_heap_up_to(target_stop)) {
|
|
return -1; /* XXX */
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
inline bool StackState::started() const G_NOEXCEPT
|
|
{
|
|
return this->stack_stop != nullptr;
|
|
}
|
|
|
|
inline bool StackState::main() const G_NOEXCEPT
|
|
{
|
|
return this->stack_stop == (char*)-1;
|
|
}
|
|
|
|
inline bool StackState::active() const G_NOEXCEPT
|
|
{
|
|
return this->_stack_start != nullptr;
|
|
}
|
|
|
|
inline void StackState::set_active() G_NOEXCEPT
|
|
{
|
|
assert(this->_stack_start == nullptr);
|
|
this->_stack_start = (char*)1;
|
|
}
|
|
|
|
inline void StackState::set_inactive() G_NOEXCEPT
|
|
{
|
|
this->_stack_start = nullptr;
|
|
// XXX: What if we still have memory out there?
|
|
// That case is actually triggered by
|
|
// test_issue251_issue252_explicit_reference_not_collectable (greenlet.tests.test_leaks.TestLeaks)
|
|
// and
|
|
// test_issue251_issue252_need_to_collect_in_background
|
|
// (greenlet.tests.test_leaks.TestLeaks)
|
|
//
|
|
// Those objects never get deallocated, so the destructor never
|
|
// runs.
|
|
// It *seems* safe to clean up the memory here?
|
|
if (this->_stack_saved) {
|
|
this->free_stack_copy();
|
|
}
|
|
}
|
|
|
|
inline intptr_t StackState::stack_saved() const G_NOEXCEPT
|
|
{
|
|
return this->_stack_saved;
|
|
}
|
|
|
|
inline char* StackState::stack_start() const G_NOEXCEPT
|
|
{
|
|
return this->_stack_start;
|
|
}
|
|
|
|
|
|
inline StackState StackState::make_main() G_NOEXCEPT
|
|
{
|
|
StackState s;
|
|
s._stack_start = (char*)1;
|
|
s.stack_stop = (char*)-1;
|
|
return s;
|
|
}
|
|
|
|
StackState::~StackState()
|
|
{
|
|
if (this->_stack_saved != 0) {
|
|
this->free_stack_copy();
|
|
}
|
|
}
|
|
|
|
using greenlet::Greenlet;
|
|
|
|
bool Greenlet::is_currently_running_in_some_thread() const
|
|
{
|
|
return this->stack_state.active() && !this->python_state.top_frame();
|
|
}
|
|
|
|
|
|
|
|
#endif
|