|
|
infodev:filesystem-walk(
|
|
$dir-path as xs:string,
|
|
$ticket-id as xs:string,
|
|
$function as xdmp:function,
|
|
$policy-deltas as element(info:options)?,
|
|
[$context as item()?]
|
| ) as empty-sequence() |
|
|
Summary:
This function can be used along with the infodev:ingest function to
build custom load operations.
This function recurses the specified filesystem directory and uses the
policy deltas from the specified ticket to determine which files to select for ingestion.
The specified process function can be written to modify the content of
files before calling the infodev:ingest function to load them into the
database.
|
Parameters:
|
$dir-path
:
The directory path to walk.
|
|
$ticket-id
:
The ID of the created ticket.
|
$function
:
The process function to apply to each file. This is where you can make modifications
to each file before loading them into the database. The function signature must be
as follows:
declare function my:function(
$document as node()?,
$source-location as xs:string,
$ticket-id as xs:string,
$policy-deltas as element(info:options)?,
$context as item()?)
as empty-sequence()
|
|
$policy-deltas
:
One or more deltas to be applied to the policy.
|
|
$context
(optional):
User-supplied context that is passed to the process function.
|
|
Example:
xquery version "1.0-ml";
import module namespace info = "http://marklogic.com/appservices/infostudio"
at "/MarkLogic/appservices/infostudio/info.xqy";
import module namespace infodev = "http://marklogic.com/appservices/infostudio/dev"
at "/MarkLogic/appservices/infostudio/infodev.xqy";
declare function local:process-file(
$document as node()?,
$source-location as xs:string,
$ticket-id as xs:string,
$policy-deltas as element(info:options)?,
$context as item()?)
{
(: You can modify the file here :)
infodev:ingest($document, $source-location, $ticket-id)
};
let $function := xdmp:function(xs:QName("local:process-file"))
let $annotation := <info:annotation>Loading XML docs</info:annotation>
let $dir-path := "C:\mydocs\xmldocs"
let $ticket := infodev:ticket-create(
$annotation,
"myDB",
(),
() )
return infodev:filesystem-walk(
$dir-path,
$ticket,
$function,
(),
() )
(: This query does the same thing as the info:load function. You can write the
process-file function to modify each file in some manner before calling
the infodev:ingest function. :)
|
|
|
|
infodev:handle-error(
|
|
$ticket-id as xs:string,
|
|
$context as xs:string,
|
|
$error as element(error:error),
|
|
[$annotation as element(info:annotation)?],
|
|
[$error-log-level as xs:string?]
|
| ) as empty-sequence() |
|
|
|
Summary:
This function provides configuration-aware error handling. If the error is to be logged,
a infodev:error element is logged to the App-Services database,
including document source-location, error code, and other information necessary to find
and fix the error. Must be used within a try-catch block.
|
Parameters:
|
$ticket-id
:
The id of the ticket.
|
|
$context
:
The file that generated the error.
|
|
$error
:
The error node caught by MarkLogic Server in the try block.
|
|
$annotation
(optional):
The description of the error.
|
|
$error-log-level
(optional):
The error log level. The possible levels are emergency, alert, critcal,
error, warning, notice, info, config, debug, fine, finer, or finest.
The default level is info.
|
|
Example:
xquery version "1.0-ml";
import module namespace info = "http://marklogic.com/appservices/infostudio"
at "/MarkLogic/appservices/infostudio/info.xqy";
import module namespace infodev = "http://marklogic.com/appservices/infostudio/dev"
at "/MarkLogic/appservices/infostudio/infodev.xqy";
declare namespace error = "http://marklogic.com/xdmp/error";
declare function local:load-doc(
$doc
) as xs:string
{
let $document := xdmp:document-get($doc)
let $path := "/xml/my.xml"
let $annotation := <info:annotation>Adding new docs to newdocs</info:annotation>
let $ticket := infodev:ticket-create($annotation, "myDB", "default", ())
let $deltas :=
<options xmlns="http://marklogic.com/appservices/infostudio">
<uri>
<literal>http://test</literal>
<path/>
<filename/>
<literal>.</literal>
<ext/>
</uri>
</options>
let $ingestion :=
try {
infodev:ingest($document, $path, $ticket),
infodev:log-progress(
$ticket,
<info:annotation>my.xml doc inserted</info:annotation>)
} catch($e) {
infodev:handle-error($ticket, $path, $e)
}
let $_ := infodev:ticket-set-status($ticket, "completed")
return (fn:concat($path, " loaded into myDB") )
};
local:load-doc("C:\test\my.xml")
(: Logs any errors encountered when loading my.xml into the database. :)
|
|
|
|
infodev:ingest(
|
|
$document as node()?,
|
|
$path as xs:string,
|
|
$ticket-id as xs:string,
|
|
[$policy-deltas as element(info:options)?],
|
|
[$properties as element()*]
|
| ) as xs:string |
|
|
|
Summary:
This function ingests a single document into the database specified in the ticket,
according to the rules defined by the named policy and user-supplied options.
|
Parameters:
|
$document
:
The node to be inserted.
|
$path
:
The name of the document and the URI path under which it is to be
inserted in the database.
In order for the path to be included in the URI, you must have
<path/> in the uri portion of your policy:
<options xmlns="http://marklogic.com/appservices/infostudio">
<uri>
<literal/>
<path/>
<filename/>
<literal>.</literal>
<ext/>
</uri>
</options>
|
|
$ticket-id
:
The ID of the ticket created by the infodev:ticket-create function.
|
|
$policy-deltas
(optional):
An options node to be used with requests associated with this ticket. It is
merged with the effective policy at runtime.
|
|
$properties
(optional):
Any properties you want to set on the document. This does the same thing as
the xdmp:document-set-properties function.
|
|
Example:
xquery version "1.0-ml";
import module namespace infodev = "http://marklogic.com/appservices/infostudio/dev"
at "/MarkLogic/appservices/infostudio/infodev.xqy";
let $document := xdmp:document-get("C:\mydocs\xmldocs\my.xml")
let $path := "/newxml/my.xml"
let $annotation := <info:annotation>Adding my.xml doc</info:annotation>
let $ticket := infodev:ticket-create($annotation, "myDB", "default", ())
let $deltas :=
<options xmlns="http://marklogic.com/appservices/infostudio">
<uri>
<literal>http://testing</literal>
<path/>
<filename/>
<literal>.</literal>
<ext/>
</uri>
</options>
return
infodev:ingest($document, $path, $ticket. $deltas)
(: loads the my.xml file, using the default policy, into the URI,
http://testing/newxml/my.xml. :)
|
|
|
|
infodev:log-progress(
|
|
$ticket-id as xs:string,
|
|
$annotation as element(info:annotation),
|
|
[$documents-processed as xs:nonNegativeInteger?],
|
|
[$transactions-completed as xs:nonNegativeInteger?],
|
|
[$error-log-level as xs:string?]
|
| ) as empty-sequence() |
|
|
|
Summary:
This function writes log information into a ticket's progress file that is written to
the App-Services database.
|
Parameters:
|
$ticket-id
:
The id of the ticket.
|
|
$annotation
:
The string to place in the annotation element.
|
|
$documents-processed
(optional):
The value to place in the documents-processed element.
|
|
$transactions-completed
(optional):
The value to place in the transactions-completed element.
|
|
$error-log-level
(optional):
The error log level. The possible levels are emergency, alert, critcal, error, warning,
notice, info, config, debug, fine, finer, or finest. The default level is info.
|
|
Example:
xquery version "1.0-ml";
import module namespace info = "http://marklogic.com/appservices/infostudio"
at "/MarkLogic/appservices/infostudio/info.xqy";
import module namespace infodev = "http://marklogic.com/appservices/infostudio/dev"
at "/MarkLogic/appservices/infostudio/infodev.xqy";
let $annotation := <info:annotation>Load took longer than 10 minutes</info:annotation>
for $ticket in info:tickets()
let $time := fn:data(info:ticket($ticket)//info:time-consumed)
return
if (fn:minutes-from-duration(xs:duration($time)) gt 10)
then infodev:log-progress($ticket, $annotation)
else ()
(: Change progress annotation on any ticket with a time-consumed value
over 10 minutes. :)
|
|
|
|
infodev:ticket-set-status(
|
|
$ticket-id as xs:string,
|
|
$status as xs:string
|
| ) as empty-sequence() |
|
|
|
Summary:
This function sets the status of the specified ticket.
|
Parameters:
|
$ticket-id
:
The id of the ticket to which to set the status.
|
|
$status
:
The status to set on the ticket. The valid status values are "aborted",
"active", "cancelled", "cleared", "completed", "inactive", "unloading", "unloaded",
and "unload-aborted".
|
|
Example:
xquery version "1.0-ml";
import module namespace infodev = "http://marklogic.com/appservices/infostudio/dev"
at "/MarkLogic/appservices/infostudio/infodev.xqy";
infodev:ticket-set-status("/tickets/ticket/13580249356496860619", "active")
(: Sets the status of the specified ticket to "active." :)
|
|
|
|
infodev:transaction(
|
|
$document-map as map:map,
|
|
$ticket-id as xs:string,
|
|
$function as xdmp:function,
|
|
$policy-deltas as element(info:options)?,
|
|
$transaction-index as xs:integer,
|
|
$context as item()?,
|
|
[$error-log-level as xs:string?]
|
| ) as empty-sequence() |
|
|
|
Summary:
This function ingests a batch of documents in a single invoked transaction.
The batch of documents is contained in $document-map.
Ingestion is done using a function named by the $function parameter.
|
Parameters:
|
$document-map
:
A map containing a batch of documents to be ingested in a single transaction.
Map keys are used to construct the URI used to locate the document
to be ingested, such as the $source-location in the function referenced
by the $function parameter, below.
|
|
$ticket-id
:
The ID of the created ticket.
|
$function
:
The ingestion function to apply to each file. This function should call infodev:ingest to
insert the document into the database.
The function signature of $function must be of the pattern:
declare function my:function(
$document as node(),
$source-location as xs:string,
$ticket-id as xs:string,
$policy-deltas as element(info:options)?,
$context as item()?)
as xs:string+
|
|
$policy-deltas
:
One or more deltas to be applied to the policy.
|
|
$transaction-index
:
An integer representing this transaction's index in a sequence of transactions.
This is used in error reporting, to distinguish between transactions.
|
|
$context
:
User-supplied context that is passed to the process function.
|
|
$error-log-level
(optional):
The error log level. The possible levels are emergency, alert, critcal,
error, warning, notice, info, config, debug, fine, finer, or finest.
The default level is info.
|
|
Example:
xquery version "1.0-ml";
import module namespace info = "http://marklogic.com/appservices/infostudio"
at "/MarkLogic/appservices/infostudio/info.xqy";
import module namespace infodev = "http://marklogic.com/appservices/infostudio/dev"
at "/MarkLogic/appservices/infostudio/infodev.xqy";
(: Define a process-file function. :)
declare function local:process-file(
$document as node()?,
$source-location as xs:string,
$ticket-id as xs:string,
$policy-deltas as element(info:options)?,
$context as item()?)
{
(: You can modify the file here :)
infodev:ingest($document, $source-location, $ticket-id)
};
let $function := xdmp:function(xs:QName("local:process-file"))
let $annotation := <info:annotation>Loading XML Shakespeare docs</info:annotation>
(: Create a ticket to load contents into the myDB database. :)
let $ticket := infodev:ticket-create($annotation, "myDB", (), ())
(: Gather the documents to be loaded into $entries. :)
let $playdir := "C:\Shakespeare\bill_xml"
let $entries := for $x in xdmp:filesystem-directory($playdir)//dir:pathname/text()
[ends-with(lower-case(.), ".xml")]
return xdmp:document-get($x)
(: Determine the number of documents; the number of documents to be loaded in each
transaction, and the total number of transactions needed to load all of the documents. :)
let $entry-count := fn:count($entries)
let $transaction-size := 5 (: Typically, this value would be extracted from the policy. :)
let $total-transactions := ceiling($entry-count div $transaction-size)
(: Create a map containing the documents to be loaded. :)
let $transactions :=
for $i at $index in 1 to $total-transactions
let $map := map:map()
let $start := (($i -1) *$transaction-size) + 1
let $finish := min((($start - 1 + $transaction-size),$entry-count))
let $put := for $entry in ($entries)[$start to $finish]
(: Create an ID for each document that consists of the date, a number, and a
filename contructed from the title of the play, minus spaces. :)
let $title := fn:replace($entry/PLAY/TITLE, " ", "")
let $id := fn:concat("/", fn:format-dateTime(fn:current-dateTime(),
"[Y01]-[M01]-[D01]_[H01]-[m01]-[s01]-[f01]") ,
"/", $index ,"/", $title, ".xml")
return map:put($map,$id,$entry)
return $map
(: Create the transactions to load the documents into the database. :)
let $ingestion :=
for $transaction at $index in $transactions
return
try {
infodev:transaction($transaction, $ticket, $function, (), $index,(),())
} catch($e) {
infodev:handle-error($ticket, concat("transaction ",$index), $e)
}
(: When the load transactions have completed, set the ticket status to "completed". :)
let $_ := infodev:ticket-set-status($ticket, "completed")
return ()
(: This query creates multiple transactions to load the XML files located in the
"C:\Shakespeare\bill_xml" directory into the myDB database. Five documents are
loaded in each transaction. :)
|
|
|