Added table create/delete/upload CADC prototype

CHANGES.rst

@@ -1,5 +1,6 @@
 1.4 (unreleased)
 ================
+- Added the TAP Table Manipulation prototype (cadc-tb-upload). [#274]
 
 - we now ignore namespaces in xsi-type attributes; this is a lame fix
   for services like ESO's and MAST's TAP, which do not use canonical

docs/dal/index.rst

@@ -186,6 +186,57 @@ The uploaded tables will be available as ``TAP_UPLOAD.name``.
   The supported upload methods are available under
   :py:meth:`~pyvo.dal.tap.TAPService.upload_methods`.
 
+.. _table manipulation:
+
+Table Manipulation
+^^^^^^^^^^^^^^^^^^
+
+.. note::
+    This is a prototype implementation and the interface might not be stable.
+    More details about the feature at: :ref:`cadc-tb-upload`
+
+Some services allow users to create, modify and delete tables. Typically, these
+functionality is only available to authenticated (and authorized) users.
+
+.. Requires proper credentials and authorization
+.. doctest-skip::
+
+    >>> auth_session = vo.auth.AuthSession()
+    >>> # authenticate. For ex: auth_session.credentials.set_client_certificate('<cert_file>')
+    >>> tap_service = vo.dal.TAPService("https://ws-cadc.canfar.net/youcat", auth_session)
+    >>>
+    >>> table_definition = '''
+    >>> <vosi:table xmlns:vosi="http://www.ivoa.net/xml/VOSITables/v1.0" xmlns:vod="http://www.ivoa.net/xml/VODataService/v1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" type="output">
+    >>>     <name>my_table</name>
+    >>>     <description>This is my very own table</description>
+    >>>     <column>
+    >>>         <name>article</name>
+    >>>         <description>some article</description>
+    >>>         <dataType xsi:type="vod:VOTableType" arraysize="30*">char</dataType>
+    >>>     </column>
+    >>>     <column>
+    >>>         <name>count</name>
+    >>>         <description>how many</description>
+    >>>         <dataType xsi:type="vod:VOTableType">long</dataType>
+    >>>     </column>
+    >>> </vosi:table> '''
+    >>> tap_service.create_table('test_schema.test_table', StringIO(table_definition))
+
+Table content can be loaded from a file or from memory. Supported data formats:
+tab-separated values (tsv), comma-separated values (cvs) or VOTable (VOTable):
+
+    >>> tap_service.load_table('test_schema.test_table',
+    >>>                        StringIO('article,count\narticle1,10\narticle2,20\n'), 'csv')
+
+Users can also create indexes on single columns:
+
+    >>> tap_service.create_index('test_schema.test_table', 'article', unique=True)
+
+Finally, tables and their content can be removed:
+
+    >>> tap_service.remove_table('test_schema.test_table')
+
+
 .. _pyvo-sia:
 
 Simple Image Access

docs/prototypes/index.rst

@@ -76,7 +76,7 @@ Feature Registry
 ================
 
 The feature registry is a static ``features`` dictionary in the `~pyvo.utils.prototype` package. The key is the name
-of the feature and the value is an instance of the `~pyvo.utils.prototype.Feature` class. This class is responsible for determining
+of the feature and the value is an instance of the `~pyvo.utils.protofeature.Feature` class. This class is responsible for determining
 whether an instance should error or not, and to format an error message if it's not. While the current implementation
 of the ``Feature`` class is simple, future requirements might lead to other implementations with more complex logic or
 additional documentation elements.
@@ -87,3 +87,23 @@ Reference/API
 =============
 
 .. automodapi:: pyvo.utils.prototype
+
+
+Existing Prototypes
+===================
+
+.. _cadc-tb-upload:
+
+CADC Table Manipulation (cadc-tb-upload)
+----------------------------------------
+
+This is a proposed extension to the TAP protocol to allow users to manipulate
+tables (https://wiki.ivoa.net/twiki/bin/view/IVOA/TAP-1_1-Next). The
+`~pyvo.dal.tap.TAPService` has been extended with methods that allow for:
+
+* table creation
+* column index creation
+* table content upload
+* table removal
+
+More details at: :ref:`table manipulation`

pyvo/auth/authsession.py

@@ -69,6 +69,18 @@ def post(self, url, **kwargs):
         """
         return self._request('POST', url, **kwargs)
 
+    def put(self, url, **kwargs):
+        """
+        Wrapper to make a HTTP PUT request with authentication.
+        """
+        return self._request('PUT', url, **kwargs)
+
+    def delete(self, url, **kwargs):
+        """
+        Wrapper to make a HTTP DELETE request with authentication.
+        """
+        return self._request('DELETE', url, **kwargs)
+
     def _request(self, http_method, url, **kwargs):
         """
         Make an HTTP request with authentication.

pyvo/dal/tap.py

@@ -23,6 +23,7 @@
 
 from ..utils.formatting import para_format_desc
 from ..utils.http import use_session
+from ..utils.prototype import prototype_feature
 import xml.etree.ElementTree
 import io
 
@@ -31,6 +32,14 @@
 
 IVOA_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 
+# file formats supported by table upload and their corresponding MIME types
+TABLE_UPLOAD_FORMAT = {'tsv': 'text/tab-separated-values',
+                       'csv': 'text/csv',
+                       'FITSTable': 'application/fits'}
+# file formats supported by table create and their corresponding MIME types
+TABLE_DEF_FORMAT = {'VOSITable': 'text/xml',
+                    'VOTable': 'application/x-votable+xml'}
+
 
 def _from_ivoa_format(datetime_str):
     """
@@ -101,7 +110,7 @@ class TAPService(DALService, AvailabilityMixin, CapabilityMixin):
 
     def __init__(self, baseurl, session=None):
         """
-        instantiate a Tablee Access Protocol service
+        instantiate a Table Access Protocol service
 
         Parameters
         ----------
@@ -115,7 +124,7 @@ def __init__(self, baseurl, session=None):
         # Check if the session has an update_from_capabilities attribute.
         # This means that the session is aware of IVOA capabilities,
         # and can use this information in processing network requests.
-        # One such usecase for this is auth.
+        # One such use case for this is auth.
         if hasattr(self._session, 'update_from_capabilities'):
             self._session.update_from_capabilities(self.capabilities)
 
@@ -451,6 +460,127 @@ def describe(self, width=None):
             cap.describe()
             print()
 
+    @prototype_feature('cadc-tb-upload')
+    def create_table(self, name, definition, format='VOSITable'):
+        """
+        Creates a table in the catalog service.
+
+        Parameters
+        ----------
+        name: str
+            Name of the table in the TAP service
+        definition: stream (object with a read method)
+            Definition of the table
+        format: str
+            Format of the table definition (VOSITable or VOTable).
+        """
+        if not name or not definition:
+            raise ValueError(
+                'table name and definition required in create: {}/{}'.
+                format(name, definition))
+        if format not in TABLE_DEF_FORMAT.keys():
+            raise ValueError(
+                'Table definition file format {} not supported ({})'.
+                format(format, ' '.join(TABLE_DEF_FORMAT.keys())))
+
+        headers = {'Content-Type': TABLE_DEF_FORMAT[format]}
+        response = self._session.put('{}/tables/{}'.format(self.baseurl, name),
+                                     headers=headers,
+                                     data=definition)
+        response.raise_for_status()
+
+    @prototype_feature('cadc-tb-upload')
+    def remove_table(self, name):
+        """
+        Remove a table from the catalog service (Equivalent to drop command
+        in DB).
+
+        Parameters
+        ----------
+        name: str
+            Name of the table in the TAP service
+        """
+        if not name:
+            raise ValueError(
+                'table name required in : {}'.
+                format(name))
+
+        response = self._session.delete(
+            '{}/tables/{}'.format(self.baseurl, name))
+        response.raise_for_status()
+
+    @prototype_feature('cadc-tb-upload')
+    def load_table(self, name, source, format='tsv'):
+        """
+        Loads content to a table
+
+        Parameters
+        ----------
+        name: str
+            Name of the table
+        source: stream with a read method
+            Stream containing the data to be loaded
+        format: str
+            Format of the data source: tab-separated values(tsv),
+            comma-separated values (csv) or FITS table (FITSTable)
+        """
+        if not name or not source:
+            raise ValueError(
+                'table name and source required in upload: {}/{}'.
+                format(name, source))
+        if format not in TABLE_UPLOAD_FORMAT.keys():
+            raise ValueError(
+                'Table content file format {} not supported ({})'.
+                format(format, ' '.join(TABLE_UPLOAD_FORMAT.keys())))
+
+        headers = {'Content-Type': TABLE_UPLOAD_FORMAT[format]}
+        response = self._session.post(
+            '{}/load/{}'.format(self.baseurl, name),
+            headers=headers,
+            data=source)
+        response.raise_for_status()
+
+    @prototype_feature('cadc-tb-upload')
+    def create_index(self, table_name, column_name, unique=False):
+        """
+        Creates a table index in the catalog service.
+
+        Parameters
+        ----------
+        table_name: str
+            Name of the table
+        column_name: str
+            Name of the column in the table
+        unique: bool
+            True for unique index, False otherwise
+        """
+        if not table_name or not column_name:
+            raise ValueError(
+                'table and column names are required in index: {}/{}'.
+                format(table_name, column_name))
+
+        result = self._session.post('{}/table-update'.format(self.baseurl),
+                                    data={'table': table_name,
+                                          'index': column_name,
+                                          'unique': 'true' if unique
+                                          else 'false'},
+                                    allow_redirects=False)
+
+        if result.status_code == 303:
+            job_url = result.headers['Location']
+            if not job_url:
+                raise RuntimeError(
+                    'table update job location missing in response')
+            # run the job
+            job = AsyncTAPJob(job_url, session=self._session)
+            job = job.run().wait()
+            job.raise_if_error()
+            # TODO job.delete()
+        else:
+            raise RuntimeError(
+                'BUG: table update expected status 303 received {}'.
+                format(result.status_code))
+
 
 class AsyncTAPJob:
     """