From: Matthew Dillon Date: Sat, 24 Mar 2007 19:11:15 +0000 (+0000) Subject: Reformulate the syslink_msg structure a bit. Instead of trying to create X-Git-Tag: v2.0.1~3349 X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/4a83d324fa7ce9048350fa02a7c84194e4384315 Reformulate the syslink_msg structure a bit. Instead of trying to create unique sessions with a combination of logical and transaction ids, which when all said and done is not very robust and not easy to index, create a 'session' abstraction which can be used as a rendezvous for a transaction as well as recorded by the route node for tracking and caching purposes. The 64 bit session ids are cluster-wide unique numbers, unique for all-time (which means 'for long enough'... i.e. at least a year), which identify abstracted connections between entities... almost the equivalent of an 'open'. The universal uniqueness of the number combined with the logical end-point tracking greatly improves robustness in the face of an ever-changing dynamic cluster mesh and guarentees that a syslink message will never be misinterpreted. Stale sessions can now be trivially detected and acted upon. This also allows us to reformulate the sysids we store in the message structure into something that is far more suitable for tracking and routing purposes without having to bloat the structure (three 64 bit ids in the structure is plenty already!). --- diff --git a/sys/kern/kern_syslink.c b/sys/kern/kern_syslink.c index 3933fbb27c..5f579f0218 100644 --- a/sys/kern/kern_syslink.c +++ b/sys/kern/kern_syslink.c @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/kern/kern_syslink.c,v 1.6 2007/03/21 20:06:34 dillon Exp $ + * $DragonFly: src/sys/kern/kern_syslink.c,v 1.7 2007/03/24 19:11:14 dillon Exp $ */ /* * This module implements the syslink() system call and protocol which @@ -355,7 +355,7 @@ syslink_rthread(void *arg) struct sldata *sldata = arg; struct slbuf *slbuf = &sldata->rbuf; struct syslink_msg *head; - const int min_msg_size = offsetof(struct syslink_msg, sm_srcid); + const int min_msg_size = SL_MIN_MESSAGE_SIZE; while ((sldata->flags & SLF_RQUIT) == 0) { int count; @@ -430,6 +430,8 @@ syslink_rthread(void *arg) /* * Process non-pad messages. Non-pad messages have to be at * least the size of the syslink_msg structure. + * + * A PAD message's sm_cmd field contains 0. */ if (head->sm_cmd) { if (head->sm_bytes < sizeof(struct syslink_msg)) { @@ -483,11 +485,11 @@ syslink_wthread(void *arg) int count; used = slbuf->windex - slbuf->rindex; - if (used < offsetof(struct syslink_msg, sm_srcid)) + if (used < SL_MIN_MESSAGE_SIZE) break; head = (void *)(slbuf->buf + (slbuf->rindex & slbuf->bufmask)); - if (head->sm_bytes < offsetof(struct syslink_msg, sm_srcid)) { + if (head->sm_bytes < SL_MIN_MESSAGE_SIZE) { error = EINVAL; break; } @@ -805,7 +807,7 @@ static int syslink_validate(struct syslink_msg *head, int bytes) { - const int min_msg_size = offsetof(struct syslink_msg, sm_srcid); + const int min_msg_size = SL_MIN_MESSAGE_SIZE; int aligned_reclen; while (bytes) { diff --git a/sys/sys/syslink_msg.h b/sys/sys/syslink_msg.h index 79920cf709..e5e8a94ab3 100644 --- a/sys/sys/syslink_msg.h +++ b/sys/sys/syslink_msg.h @@ -31,7 +31,7 @@ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $DragonFly: src/sys/sys/syslink_msg.h,v 1.4 2007/03/21 20:06:36 dillon Exp $ + * $DragonFly: src/sys/sys/syslink_msg.h,v 1.5 2007/03/24 19:11:15 dillon Exp $ */ /* * The syslink infrastructure implements an optimized RPC mechanism across a @@ -57,9 +57,8 @@ #include #endif -typedef u_int32_t sl_cookie_t; typedef u_int32_t sl_msgid_t; -typedef u_int16_t sl_cid_t; /* command or error */ +typedef u_int16_t sl_cmd_t; /* command or error */ typedef u_int16_t sl_itemid_t; /* item id */ typedef u_int16_t sl_reclen_t; /* item length */ @@ -75,89 +74,90 @@ typedef u_int16_t sl_reclen_t; /* item length */ * headers (i.e. it does not represent just the data payload, it represents * the entire structure). * - * A message transaction consists of a command message and a reply message. - * A message is uniquely identified by (src_sysid, dst_sysid, msgid). Note - * that both ends can initiate a unique command using the same msgid at - * the same time, there is no confusion between (src,dst,id) and (dst,src,id). - * The reply message will have bit 31 set in the msgid. Transactions occur - * over reliable links and use a single-link abstraction (even if the - * underlying topology is a mesh). Therefore, there is no need to have - * timeouts or retries. If a link is lost the transport layer is responsible - * for aborting any transactions which have not been replied to. + * Sessions are identified with a session id. The session id is a rendezvous + * id that associates physical and logical routing information with a single + * sysid, allowing us to both avoid storing the source and target logical id + * in the syslink message AND ALSO providing a unique session id for an + * abstracted connection between two entities. Otherwise the syslink message + * would become bloated with five sysid fields instead of the three we have + * now. * - * Multiple transactions may run in parallel between two entities. Only bit - * 31 of the msgid is reserved (to indicate a reply). Bits 30-0 may be - * specified by the originator. It is often convenient, but not required, - * to shift the pointer to the internal representation of the command on the - * originator a few bits and use that as the message id, removing the need - * to keep track of unique ids. - * - * Both the governing syslink_msg and embedded syslink_items are limited to - * 65535 bytes. When pure data is passed in a message it is typically - * limited to 32KB. If transport is via a memory-FIFO or might pass through - * a memory-FIFO, pad is usually added to prevent a message from wrapping - * around the FIFO boundary, allowing the FIFO to be memory mapped. This - * is typically negotiated by the transport layer. - * - * In a command initiation the cid is the command. The command format is - * not specified over the wire (other then 0x0000 being reserved), but - * typically the high 8 bits specify a protocol and the low 8 bits specify - * an index into the protocol's function array. Even though most sysid's - * implement only a single protocol and overloading is possible, using - * unique command id's across all protocols can be thought of as a safety. - * - * The cid field represents an error code in the response. - * - * The source and destination sysid is relative to the originator. The - * transport layer checks the high bits of the source sysid matches its - * expectations. These bits may be shortcut to a 0. If the message is - * transported to a non-local domain the transport layer must replace the - * 0 with the correct coordinates within the topology (insofar as it knows - * them). Since locality is layered not all high bits have to be replaced. - * The transport for a higher layer will continue to adjust segments of 0's - * as required to identify the message's source. - * - * PAD messages may be inserted in the stream by the transport layer(s) and - * are always stripped on the receiving end. However, they may be visible - * to the receiver (e.g. if the transport is a memory mapped FIFO). If - * multiple transport hops occur PAD is typically reformulated and it is - * up to the transport layer to decide whether it has to strip and - * reformulate it, or whether it can just pass it through, or negotiate - * a buffer alignment such that no reformulation is required. - * - * A PAD message uses a msgid of 0. As a special case, PAD messages can be - * as small as 8 bytes (msgid, cid, and reclen fields only). The sysid - * fields, if present, are ignored. + * Link layer communications is accomplished by specifying a session id of + * 0. */ /* * Raw protocol structures */ struct syslink_msg { - u_int16_t sm_cmd; /* protocol command code */ - u_int16_t sm_bytes; /* unaligned size of message */ - u_int32_t sm_seq; + sl_cmd_t sm_cmd; /* protocol command code */ + sl_reclen_t sm_bytes; /* unaligned size of message */ + sl_msgid_t sm_msgid; /* message transaction control */ /* minimum syslink_msg size is 8 bytes (special PAD) */ - sysid_t sm_srcid; /* origination logical sysid */ - sysid_t sm_dstid; /* destination logical sysid */ - sysid_t sm_dstpysid; /* cached physical sysid */ -}; - -struct syslink_elm { - u_int16_t se_ctl; - u_int16_t se_bytes; - u_int32_t se_reserved; - /* extended by data */ + sysid_t sm_sessid; /* session id */ + sysid_t sm_srcphysid; /* originating physical id */ + sysid_t sm_dstphysid; /* target physical id */ }; +#define SL_MIN_MESSAGE_SIZE offsetof(struct syslink_msg, sm_sessid) + +#define SL_MSGID_REPLY 0x80000000 /* command vs reply */ +#define SL_MSGID_ORIG 0x40000000 /* originator transaction */ + /* (else target transaction) */ +#define SL_MSGID_BEG 0x20000000 /* first msg in transaction */ +#define SL_MSGID_END 0x10000000 /* last msg in transaction */ +#define SL_MSGID_STRUCTURED 0x08000000 /* contains structured data */ +#define SL_MSGID_TRANS_MASK 0x00FFFF00 /* transaction id */ +#define SL_MSGID_SEQ_MASK 0x000000FF /* sequence no within trans */ + #define SLMSG_ALIGN(bytes) (((bytes) + 7) & ~7) -#define SLMSGF_RESPONSE ((sl_msgid_t)0x80000000) -#define SLIF_RECURSION ((sl_cid_t)0x8000) -#define SLIF_REFID ((sl_cid_t)0x4000) +/* + * Syslink message commands (16 bits, bit 15 must be 0) + * + * Commands 0x0000-0x001F are reserved for the universal link layer, but + * except for 0x0000 (which is a PAD message), must still be properly + * routed. + * + * Commands 0x0020-0x002F are reserved for the universal protocol + * identification layer. + * + * Commands 0x0100-0x7FFF are protocol commands. + * + * The command field is the error return field with bit 15 set in the + * reply. + */ +#define SL_CMD_PAD 0x0000 +#define SL_CMD_LINK_MESH 0x0001 /* mesh construction */ +#define SL_CMD_LINK_REG 0x0002 /* register logical id */ +#define SL_CMD_LINK_DEREG 0x0003 /* unregister logical id */ +#define SL_CMD_LINK_ID 0x0004 /* link level identification */ + +#define SL_CMD_PROT_ID 0x0010 /* protocol & device ident */ + +/* + * Message elements for structured messages. If SL_MSGID_STRUCTURED is + * set the syslink message contains zero or more syslink_elm structures + * laid side by side. Each syslink_elm structure may or may not be + * structured (i.e. recursive). + * + * Most of the same SL_MSGID_* bits apply. The first structured element + * will have SL_MSGID_BEG set, the last will have SL_MSGID_END set (if + * there is only one element, both bits will be set in that element). If + * the payload is structured, SL_MSGID_STRUCTURED will be set. + * + * syslink_elm's may use the TRANS and SEQ bits in the msgid for other + * purposes. A syslink_elm is considered to be a PAD if se_cmd == 0. + */ +struct syslink_elm { + sl_cmd_t se_cmd; + sl_reclen_t se_bytes; + sl_msgid_t se_msgid; + /* extended by data */ +}; typedef struct syslink_msg *syslink_msg_t; -typedef struct syslink_item *syslink_item_t; +typedef struct syslink_elm *syslink_elm_t; #endif