Probable questions

#include	"unp.h"

void  str_echo(int sockfd);
void  sig_chld(int signo);

int
main(int argc, char **argv)
{
    int     listenfd, connfd;
    pid_t   childpid;
    socklen_t clilen;
    struct sockaddr_in cliaddr, servaddr;

    listenfd = Socket(AF_INET, SOCK_STREAM, 0);

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family      = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port        = htons(SERV_PORT);

    Bind(listenfd, (SA *) &servaddr, sizeof(servaddr));
    Listen(listenfd, LISTENQ);

    Signal(SIGCHLD, sig_chld);          /* reap children */

    for ( ; ; ) {
        clilen = sizeof(cliaddr);
        if ( (connfd = accept(listenfd, (SA *) &cliaddr, &clilen)) < 0) {
            if (errno == EINTR)
                continue;                  /* interrupted by SIGCHLD */
            else
                err_sys("accept error");
        }

        if ( (childpid = Fork()) == 0) {   /* child */
            Close(listenfd);
            str_echo(connfd);
            exit(0);
        }
        Close(connfd);                     /* parent */
    }
}

void
sig_chld(int signo)
{
    pid_t  pid;
    int    stat;

    while ( (pid = waitpid(-1, &stat, WNOHANG)) > 0)
        ;                                 /* loop: signals are not queued */
    return;
}

void
str_echo(int sockfd)
{
    ssize_t n;
    char    buf[MAXLINE];

again:
    while ( (n = read(sockfd, buf, MAXLINE)) > 0)
        Writen(sockfd, buf, n);

    if (n < 0 && errno == EINTR)
        goto again;
    else if (n < 0)
        err_sys("str_echo: read error");
}

The four things that carry the marks

#include	"unp.h"
#include	<netinet/tcp.h>

int
main(int argc, char **argv)
{
    int        sockfd, optval;
    socklen_t  optlen;
    struct linger  ling;

    sockfd = Socket(AF_INET, SOCK_STREAM, 0);

    /* --- a VALUE option: receive buffer size --- */
    optlen = sizeof(optval);
    Getsockopt(sockfd, SOL_SOCKET, SO_RCVBUF, &optval, &optlen);
    printf("SO_RCVBUF    = %d bytes\n", optval);

    optlen = sizeof(optval);
    Getsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &optval, &optlen);
    printf("SO_SNDBUF    = %d bytes\n", optval);

    /* --- a GET-ONLY option: the socket's type --- */
    optlen = sizeof(optval);
    Getsockopt(sockfd, SOL_SOCKET, SO_TYPE, &optval, &optlen);
    printf("SO_TYPE      = %s\n",
           optval == SOCK_STREAM ? "SOCK_STREAM" : "SOCK_DGRAM");

    /* --- a FLAG option: zero means off, nonzero means on --- */
    optlen = sizeof(optval);
    Getsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen);
    printf("SO_KEEPALIVE = %s\n", optval ? "ON" : "OFF");

    /* now turn that flag ON and read it back */
    optval = 1;
    Setsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &optval, sizeof(optval));

    optlen = sizeof(optval);
    Getsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen);
    printf("after set    = %s\n", optval ? "ON" : "OFF");

    /* --- an option carrying a STRUCTURE --- */
    optlen = sizeof(ling);
    Getsockopt(sockfd, SOL_SOCKET, SO_LINGER, &ling, &optlen);
    printf("SO_LINGER    = l_onoff %d, l_linger %d\n",
           ling.l_onoff, ling.l_linger);

    Close(sockfd);
    exit(0);
}

What to say about it

• Reset optlen before every getsockopt. It is a value-result argument — going in it says how big your variable is, coming out it says how much the kernel stored. Setting it once at the top is the same bug as hoisting clilen out of an accept loop.
• The two categories are visible in the code. A flag option reads back zero for disabled and non-zero for enabled, and you set it with a non-zero int. A value option carries an actual quantity — and SO_LINGER carries a whole struct linger.
• SO_TYPE is get-only. Attempting to setsockopt it fails — the type was fixed when socket() was called.
• Note the asterisk difference: setsockopt takes socklen_t optlen, getsockopt takes socklen_t *optlen.

Any "client sends X, server returns f(X)" question is the echo pair with one function swapped. Last year it was the string's length; reversing, upper-casing and counting words are the other usual asks.

Server

#include	"unp.h"

void  str_reverse(int sockfd);

int
main(int argc, char **argv)
{
    int     listenfd, connfd;
    socklen_t clilen;
    struct sockaddr_in cliaddr, servaddr;
    char    buff[MAXLINE];

    listenfd = Socket(AF_INET, SOCK_STREAM, 0);

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family      = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port        = htons(SERV_PORT);

    Bind(listenfd, (SA *) &servaddr, sizeof(servaddr));
    Listen(listenfd, LISTENQ);

    for ( ; ; ) {
        clilen = sizeof(cliaddr);         /* value-result: reset each time */
        connfd = Accept(listenfd, (SA *) &cliaddr, &clilen);

        printf("connection from %s, port %d\n",
               Inet_ntop(AF_INET, &cliaddr.sin_addr, buff, sizeof(buff)),
               ntohs(cliaddr.sin_port));

        if (Fork() == 0) {
            Close(listenfd);
            str_reverse(connfd);
            exit(0);
        }
        Close(connfd);
    }
}

void
str_reverse(int sockfd)
{
    ssize_t  n;
    char     line[MAXLINE], out[MAXLINE];
    int      i, len;

    for ( ; ; ) {
        if ( (n = Readline(sockfd, line, MAXLINE)) == 0)
            return;                       /* client closed */

        line[strcspn(line, "\r\n")] = '\0';   /* strip terminator */
        len = strlen(line);

        for (i = 0; i < len; i++)
            out[i] = line[len - 1 - i];
        out[len]     = '\n';
        out[len + 1] = '\0';

        Writen(sockfd, out, strlen(out));
    }
}

Client

Identical to the echo client — socket, connect, then a loop of Fgets / Writen / Readline / Fputs. See question 12; nothing changes.

The OSI (Open Systems Interconnection) model was described by ISO for computer communications. It provides a detailed standard for describing a network, in seven layers. "Open" originally meant that networks could be interconnected regardless of the underlying hardware as long as the software adhered to the standards; it has since come to imply modularity as well.

Application-layer protocols to name

HTTP, SMTP, FTP, TELNET, SSH, SNMP, DNS, IGMP.

Network-layer helpers to name

• ICMP — helps IP report problems when routing a packet.
• IGMP — helps IP in multicasting.
• DHCP — helps IP get the network-layer address for a host.
• ARP — finds the link-layer address of a host or router given its network-layer address.

Where sockets sit — the part the question is really asking

In the Internet protocol suite the upper three OSI layers are combined into a single layer called the application, because with Internet protocols there is rarely any distinction between them. The sockets API is the interface from those upper three layers down into the transport layer, and that is the boundary this whole course lives on.

Two reasons the boundary is drawn there:

1. The upper three layers handle all the details of the application — FTP, Telnet, HTTP — and know little about communication.
2. The lower four know little about the application but handle all the communication details: sending data, waiting for acknowledgments, sequencing data that arrives out of order, calculating and verifying checksums.

A raw socket lets the application bypass the transport layer and talk to IP or even the datalink layer directly — the gap drawn between TCP and UDP in the figure. TLI (Transport Layer Interface, from AT&T System V) and its successor XTI (X/Open Transport Interface) are an alternative API at the same boundary.

Advantages of layering — five bullets, one mark

• Simplifies learning — processes are broken into groups, dividing complexity into manageable chunks.
• Reduces complexity — implementation of the architecture is less complex.
• Provides compatibility — standardised interfaces allow plug-and-play and multi-vendor integration.
• Facilitates modularisation — developers swap in new technologies at one layer without breaking the architecture.
• Accelerates evolution — work at one layer is prevented from affecting another.

Full chapter with the diagram →

1. The server must be prepared to accept an incoming connection — normally socket, bind, listen. This is a passive open.
2. The client issues an active open by calling connect. The client TCP sends a synchronise (SYN) segment telling the server the client's initial sequence number J for the data it will send. Normally no data is sent with the SYN — just an IP header, a TCP header and possible options.
3. The server must acknowledge the client's SYN and send its own SYN containing its initial sequence number K. It sends both in a single segment.
4. The client must acknowledge the server's SYN.

Why three and not four

Logically four things happen — the client's SYN, its acknowledgment, the server's SYN, and its acknowledgment. Three packets suffice because the server combines its SYN and the ACK of the client's SYN into one segment. Hence "the minimum number of packets required for this exchange is three; hence, this is called TCP's three-way handshake".

Why the acknowledgment numbers are J+1 and K+1

The acknowledgment number in an ACK is the next expected sequence number for the end sending the ACK. Since a SYN occupies one byte of sequence number space, the acknowledgment of each SYN is the initial sequence number plus one. The same rule applies to a FIN.

TCP options carried on the SYN

• MSS option — the sender announces its maximum segment size, the maximum data it will accept per segment. The sending TCP uses the receiver's MSS as its own maximum. Accessible via TCP_MAXSEG.
• Window scale option — the advertised window field is 16 bits, so the maximum is 65,535. High-speed connections (45 Mbits/sec and faster) and long-delay paths (satellite) need more. This option left-shifts the advertised window by 0–14 bits, giving almost one gigabyte (65,535 × 2¹⁴). Both ends must support it. Influenced by SO_RCVBUF.
• Timestamp option — needed on high-speed connections to prevent data corruption from old, delayed or duplicated segments.

The last two are called the RFC 1323 options or the long fat pipe options.

The telephone analogy — good for a closing paragraph

socket is having a telephone to use. bind is telling people your number. listen is turning the ringer on. connect requires knowing the other number and dialling it. accept is answering. Having the client's identity returned by accept is like caller ID — except that caller ID shows the number before you answer, whereas accept returns the client's identity only after the connection is established.

1. One application calls close first — that end performs the active close. Its TCP sends a FIN, meaning it is finished sending data.
2. The other end performs the passive close. The received FIN is acknowledged by TCP, and the receipt of the FIN is also passed to the application as an end-of-file, after any data already queued — since the FIN means no additional data will arrive on the connection.
3. Sometime later, the application that received the end-of-file closes its socket. This causes its TCP to send a FIN.
4. The TCP that receives this final FIN — the end that did the active close — acknowledges it.

Why four

Because a TCP connection is full-duplex: it is really two independent byte streams, and each direction must be shut down separately. A FIN and an ACK are required in each direction, so four segments are normally needed.

"Normally", because in some scenarios the FIN in step 1 is sent with data, and the segments in steps 2 and 3 are both from the passive-close end and could be combined into one.

Half-close

Between steps 2 and 3 it is possible for data to flow from the end doing the passive close to the end doing the active close. This is called a half-close, and the shutdown function exists to produce it deliberately.

Two points that earn extra marks

• Either end may perform the active close. It is often the client, but with some protocols — notably HTTP/1.0 — the server does.
• A FIN is sent whenever a socket is closed, not only by close. When a Unix process terminates — voluntarily via exit or main returning, or involuntarily on a terminating signal — all open descriptors are closed, which sends a FIN on any still-open TCP connection.

The end that performs the active close goes through the TIME_WAIT state, and the duration it remains there is twice the maximum segment lifetime — 2MSL.

MSL

Every implementation of TCP must choose a value for the MSL. RFC 1122 recommends 2 minutes; Berkeley-derived implementations traditionally use 30 seconds. So TIME_WAIT lasts between 1 and 4 minutes.

The MSL is the maximum amount of time any given IP datagram can live in a network. We know it is bounded because every datagram contains an 8-bit hop limit — the IPv4 TTL field, the IPv6 hop limit field — with a maximum value of 255. Strictly this is a hop limit, not a true time limit; the assumption is that a packet with the maximum hop limit of 255 cannot exist in a network for more than MSL seconds.

Lost duplicates

A packet gets "lost" as a result of routing anomalies. A router crashes or a link goes down, and the routing protocols take seconds or minutes to stabilise; during that time routing loops can occur (router A sends packets to B, and B sends them back to A) and packets get caught in them. Meanwhile the sending TCP times out and retransmits, and the retransmitted packet reaches the destination by an alternate path. Later the loop is corrected and the original packet finally arrives. That original is called a lost duplicate or wandering duplicate. TCP must handle these.

Reason 1 — to implement TCP's full-duplex connection termination reliably

Assume the final ACK is lost. The server will resend its final FIN, so the client must maintain state information allowing it to resend the final ACK. If it did not maintain this information it would respond with an RST — a different type of TCP segment — which the server would interpret as an error.

If TCP is doing all the work necessary to terminate both directions of data flow cleanly, it must correctly handle the loss of any of those four segments. This also explains why it is the active-close end that waits: that is the end which might have to retransmit the final ACK.

Reason 2 — to allow old duplicate segments to expire in the network

Suppose a connection between 12.106.32.254:1500 and 206.168.112.219:21 is closed, and sometime later another connection is established between the same IP addresses and ports. The latter is called an incarnation of the previous connection.

TCP must prevent old duplicates from the previous connection reappearing later and being misinterpreted as belonging to the new incarnation. To do this, TCP will not initiate a new incarnation of a connection that is currently in the TIME_WAIT state.

Why 2MSL specifically

Since the duration is twice the MSL, it allows MSL seconds for a packet in one direction to be lost, and another MSL seconds for the reply to be lost. Enforcing this rule guarantees that when a TCP connection is successfully established, all old duplicates from previous incarnations have expired in the network.

At any given time multiple processes can be using any given transport. All three transport layers — UDP, TCP and SCTP — use 16-bit integer port numbers to differentiate between these processes. The Internet Assigned Numbers Authority (IANA) maintains the list of assignments.

Well-known versus ephemeral

When a client wants to contact a server it must identify the server, so servers use well-known ports that are fixed and published. Clients use ephemeral ports — short-lived, assigned automatically by the transport protocol. A client does not care what the number is; it only needs it to be unique on the client host, and the transport protocol code guarantees that uniqueness.

Socket pair

The two values that identify one endpoint — an IP address and a port number — are together often called a socket.

Why four values are needed

Twenty browser tabs connect to the same web server: the same foreign address, the same foreign port 80, from the same local address. If a connection were identified by fewer than four values they would be indistinguishable. It is the different local (ephemeral) port that separates them. The same fact is how a concurrent server keeps hundreds of clients apart while every one of them is talking to the same well-known port.

SCTP and UDP

• SCTP: an association is identified by a set of local IP addresses, a local port, a set of foreign IP addresses, and a foreign port. Where neither endpoint is multihomed this reduces to the same four-tuple as TCP; but when either endpoint is multihomed, multiple four-tuple sets — different IP addresses, the same port numbers — may identify the same association.
• UDP: the concept extends even though UDP is connectionless. bind lets the application specify the local IP address and local port for TCP, UDP and SCTP alike.

Definitions

• MTU — the largest frame a given link can carry.
• Path MTU — the smallest MTU on the path between two hosts. Today the Ethernet MTU of 1,500 is often the path MTU. It need not be the same in both directions, because Internet routing is often asymmetric — the route from A to B can differ from the route from B to A.
• MSS — the maximum amount of TCP data the peer can send per segment, announced in the MSS option on each SYN.
• SCTP fragmentation point — based on the smallest path MTU found to all of the peer's addresses, a consequence of multihoming.

Fragmentation — the IPv4/IPv6 contrast

Fragments are not normally reassembled until they reach the final destination — though firewalls acting as routers sometimes reassemble so the whole packet can be inspected, at the cost of complexity and of requiring the firewall to sit on the only path. Both ICMP errors are what makes path MTU discovery work: send with DF set, and reduce the amount of data per datagram when an error comes back.

1 · IPv4 — struct sockaddr_in

Commonly called an "Internet socket address structure", named sockaddr_in and defined by including <netinet/in.h>. Sixteen bytes.

struct in_addr {
    in_addr_t   s_addr;         /* 32-bit IPv4 address, network byte order */
};

struct sockaddr_in {
    uint8_t         sin_len;    /* length of structure (16) */
    sa_family_t     sin_family; /* AF_INET */
    in_port_t       sin_port;   /* 16-bit port, network byte order */
    struct in_addr  sin_addr;   /* 32-bit IPv4 address */
    char            sin_zero[8];/* unused */
};

• sin_len — having a length field simplifies handling of variable-length socket address structures. Even if it is present you need never set it and never examine it, unless you are dealing with routing sockets; it is used inside the kernel by the routines that handle socket address structures from various protocol families.
• The four functions that pass a structure from process to kernel — bind, connect, sendto, sendmsg — go through the sockargs function in Berkeley-derived implementations, which copies the structure from the process and sets sin_len to the size passed as an argument.
• The five that pass one from kernel to process — accept, recvfrom, recvmsg, getpeername, getsockname — set sin_len before returning.
• POSIX requires only three members: sin_family, sin_addr and sin_port. Almost all implementations add sin_zero so that all socket address structures are at least 16 bytes.
• sa_family_t can be any size — normally 8-bit unsigned if the implementation supports the length field, 16-bit if not. The datatypes u_char, u_short, u_int and u_long are all unsigned and all obsolete.

2 · Generic — struct sockaddr

struct sockaddr {
    uint8_t      sa_len;
    sa_family_t  sa_family;     /* address family: AF_xxx */
    char         sa_data[14];   /* protocol-specific address */
};

Socket functions were defined before ANSI C, so there was no void *. The generic structure exists so that one prototype can accept an address from any protocol family. The consequence is that any call must cast the pointer to the protocol-specific structure to be a pointer to the generic one:

struct sockaddr_in  serv;

connect(sockfd, (struct sockaddr *) &serv, sizeof(serv));

3 · IPv6 — struct sockaddr_in6

struct sockaddr_in6 {
    uint8_t         sin6_len;      /* length of this struct (28) */
    sa_family_t     sin6_family;   /* AF_INET6 */
    in_port_t       sin6_port;     /* transport-layer port */
    uint32_t        sin6_flowinfo; /* flow information */
    struct in6_addr sin6_addr;     /* 128-bit IPv6 address */
    uint32_t        sin6_scope_id; /* scope zone identifier */
};

28 bytes, not 16 — which is precisely why struct sockaddr cannot hold one, and therefore why the next structure had to be invented.

4 · New generic — struct sockaddr_storage

Note that apart from ss_family and ss_len the rest of it is opaque — the contents must be reached by casting or copying, and its field names begin ss_, not sa_.

A 16-bit integer is two bytes, and there are two ways to store them: little-endian, with the low-order byte at the starting address, and big-endian, with the high-order byte at the starting address. The terms indicate which end of the multi-byte value — the little end or the big end — is stored at the starting address.

There is no standard between them and both are in use. Host byte order is whatever a given system uses; network byte order is the standardised big-endian format for transmission across networks. An implementation could store the fields of a socket address structure in host byte order and convert when moving them to and from protocol headers — but the API requires network byte order in the structure.

The four byte ordering functions

#include <netinet/in.h>

uint16_t  htons(uint16_t host16bitvalue);
uint32_t  htonl(uint32_t host32bitvalue);
                        Both return: value in network byte order

uint16_t  ntohs(uint16_t net16bitvalue);
uint32_t  ntohl(uint32_t net32bitvalue);
                        Both return: value in host byte order

h is host, n is network, s is short and l is long. The terms "short" and "long" are historical artefacts: think of s as a 16-bit value such as a TCP or UDP port number, and l as a 32-bit value such as an IPv4 address — because on an LP64 system a C long is 64 bits but htonl still converts 32.

Bit ordering in RFCs

A separate convention: an RFC drawing a protocol header shows the four bytes in the order in which they appear on the wire, with the leftmost bit the most significant.

The byte manipulation functions

Two groups that operate on multi-byte fields without interpreting the data and without assuming it is a null-terminated C string — necessary because an IP address can contain zero bytes but is not a C string. Functions beginning with str, from <string.h>, deal with null-terminated strings.

The full byteorder.c program and its explanation →

TCP's guarantees, in detail

• Acknowledgment and retransmission — when TCP sends data it requires an acknowledgment; if none arrives it retransmits automatically and waits a longer amount of time.
• Dynamic RTT estimation — TCP estimates the round-trip time between client and server so it knows how long to wait.
• Sequencing — a sequence number is associated with every byte sent; out-of-order segments are reordered before delivery and duplicates discarded.
• Flow control — the advertised window.
• Full-duplex.

SCTP's two additions

• Multistreaming. Multiple streams between the endpoints, each with its own reliable sequenced delivery. A lost message in one stream does not block delivery of messages in any of the others — in contrast to TCP, where a loss at any point in the single byte stream blocks all future data until the loss is repaired. That blocking is called head-of-line blocking.
• Multihoming. A single endpoint may support multiple IP addresses, giving increased robustness against network failure: an endpoint can have redundant network connections, each with a different path to the Internet, and SCTP can work around a failure by switching to another address already in the association.

Choosing — a paragraph that scores

Count the cost. For a one-segment request and a one-segment reply, TCP costs eight segments of overhead — three to set up, four to tear down, plus the acknowledgment — while UDP exchanges only two packets. Many applications are built on UDP precisely because they exchange small amounts of data and want to avoid setup and teardown; DNS is the classic case.

But switching to UDP removes all the reliability TCP provides, pushing those details into the application — including congestion control, which is the one usually forgotten. Choose TCP when the data must arrive complete and in order; UDP when the exchange is short, loss is tolerable, and latency matters; SCTP when you need TCP's reliability but also message boundaries, independent streams or path redundancy — which is why it was designed for telephony signalling.

#include <sys/socket.h>

int listen(int sockfd, int backlog);
                                Returns: 0 if OK, -1 on error

listen is called only by a TCP server and performs two actions:

1. A socket created by socket is assumed to be an active socket — a client socket that will issue a connect. listen converts an unconnected socket into a passive socket, indicating that the kernel should accept incoming connection requests directed to it. In terms of the state transition diagram, it moves the socket from CLOSED to LISTEN.
2. The second argument specifies the maximum number of connections the kernel should queue for this socket.

It is normally called after socket and bind, and must be called before accept.

The two queues

How an entry moves between them

1. When a SYN arrives, TCP creates a new entry on the incomplete queue and responds with the second segment of the handshake — its SYN with an ACK of the client's SYN. The parameters from the listening socket are copied over to the newly created connection. The connection creation mechanism is completely automatic; the server process is not involved.
2. The entry remains on the incomplete queue until the third segment arrives, or until it times out — Berkeley-derived implementations use 75 seconds.
3. If the handshake completes normally, the entry moves from the incomplete queue to the end of the completed queue.
4. When the process calls accept, the first entry on the completed queue is returned; if the queue is empty, the process is put to sleep until an entry is placed on it.

Historically the sum of both queues could not exceed backlog. The book's unp.h defines LISTENQ as 1024.

The queue diagram →

#include <sys/socket.h>

int getsockname(int sockfd, struct sockaddr *localaddr,
                socklen_t *addrlen);

int getpeername(int sockfd, struct sockaddr *peeraddr,
                socklen_t *addrlen);
                                Both return: 0 if OK, -1 on error

getsockname returns the local protocol address associated with a socket; getpeername returns the foreign one. The final argument of both is a value-result argument — these are two of the four functions on that list.

The term "name" is misleading: they return the protocol address associated with one end of a connection, which for IPv4 and IPv6 is an IP address plus a port number. They have nothing to do with domain names.

The five reasons they are required

1. After connect returns successfully in a TCP client that does not call bind, getsockname returns the local IP address and local port number assigned to the connection by the kernel.
2. After calling bind with a port number of 0 — telling the kernel to choose — getsockname returns the local port number that was assigned.
3. getsockname can be called to obtain the address family of a socket.
4. In a TCP server that binds the wildcard IP address, once a connection is established the server can call getsockname to obtain the local IP address assigned to the connection. The descriptor in this call must be the connected socket, not the listening socket.
5. When a server is execed by the process that calls accept, the only way the server can obtain the identity of the client is to call getpeername — which is what happens whenever inetd forks and execs a TCP server.

The echo server performs three steps:

1. The client reads a line of text from its standard input and writes the line to the server.
2. The server reads the line from its network input and echoes the line back to the client.
3. The client reads the echoed line and prints it on its standard output.

The server

#include	"unp.h"

int
main(int argc, char **argv)
{
    int     listenfd, connfd;
    pid_t   childpid;
    socklen_t clilen;
    struct sockaddr_in cliaddr, servaddr;

    listenfd = Socket(AF_INET, SOCK_STREAM, 0);

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family      = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port        = htons(SERV_PORT);

    Bind(listenfd, (SA *) &servaddr, sizeof(servaddr));
    Listen(listenfd, LISTENQ);

    for ( ; ; ) {
        clilen = sizeof(cliaddr);
        connfd = Accept(listenfd, (SA *) &cliaddr, &clilen);

        if ( (childpid = Fork()) == 0) {   /* child process */
            Close(listenfd);               /* close listening socket */
            str_echo(connfd);              /* process the request */
            exit(0);
        }
        Close(connfd);                     /* parent closes connected socket */
    }
}

void
str_echo(int sockfd)
{
    ssize_t n;
    char    buf[MAXLINE];

again:
    while ( (n = read(sockfd, buf, MAXLINE)) > 0)
        Writen(sockfd, buf, n);

    if (n < 0 && errno == EINTR)
        goto again;
    else if (n < 0)
        err_sys("str_echo: read error");
}

The client

#include	"unp.h"

int
main(int argc, char **argv)
{
    int     sockfd;
    struct sockaddr_in servaddr;

    if (argc != 2)
        err_quit("usage: tcpcli <IPaddress>");

    sockfd = Socket(AF_INET, SOCK_STREAM, 0);

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port   = htons(SERV_PORT);
    Inet_pton(AF_INET, argv[1], &servaddr.sin_addr);

    Connect(sockfd, (SA *) &servaddr, sizeof(servaddr));

    str_cli(stdin, sockfd);            /* do it all */

    exit(0);
}

void
str_cli(FILE *fp, int sockfd)
{
    char   sendline[MAXLINE], recvline[MAXLINE];

    while (Fgets(sendline, MAXLINE, fp) != NULL) {

        Writen(sockfd, sendline, strlen(sendline));

        if (Readline(sockfd, recvline, MAXLINE) == 0)
            err_quit("str_cli: server terminated prematurely");

        Fputs(recvline, stdout);
    }
}

Normal startup

Start the server in the background. It calls socket, bind, listen and blocks in accept. netstat -a shows one socket in LISTEN with a wildcard local address. Then start the client, which calls socket and connect, causing the three-way handshake. When it completes, connect returns in the client and accept returns in the server. Then:

1. The client calls str_cli, which blocks in fgets because no line has been typed.
2. When accept returns, the server calls fork and the child calls str_echo, which calls readline → read, blocking while waiting for a line.
3. The server parent calls accept again and blocks waiting for the next client.

Three processes, and all three are asleep. Note that connect returns when the second segment is received, but accept does not return until the third is — half an RTT later.

Normal termination — the seven steps

1. We type the EOF character (Ctrl-D). fgets returns a null pointer and str_cli returns.
2. str_cli returns to the client main, which terminates by calling exit.
3. Part of process termination is closing all open descriptors, so the client socket is closed by the kernel. This sends a FIN to the server, to which the server TCP responds with an ACK — the first half of the termination sequence. The server socket is now in CLOSE_WAIT and the client socket in FIN_WAIT_2.
4. When the server TCP receives the FIN, the server child is blocked in readline, which returns 0. str_echo returns to the child main, which terminates by calling exit.
5. All open descriptors in the child are closed. Closing the connected socket causes the final two segments: a FIN from server to client, and an ACK from client to server.
6. The connection is now completely terminated. The client socket enters TIME_WAIT.
7. SIGCHLD is sent to the parent when the server child terminates. It is not caught here, and the default action is to be ignored, so the child enters the zombie state — verifiable with ps, where its STAT is Z.

Signals

A signal is a notification to a process that an event has occurred, sometimes called a software interrupt. Signals usually occur asynchronously — a process does not know ahead of time exactly when one will occur. They can be sent by one process to another process (or to itself) and by the kernel to a process.

Every signal has a disposition, also called the action associated with it, set by calling sigaction. There are three choices:

1. Catch it — provide a function called whenever the signal occurs, a signal handler. Its prototype is always void handler(int signo);. SIGKILL and SIGSTOP cannot be caught.
2. Ignore it — set the disposition to SIG_IGN. SIGKILL and SIGSTOP cannot be ignored either.
3. Default — set it to SIG_DFL. The default is normally to terminate the process, with certain signals also generating a core image in the current working directory. A few signals default to being ignored — SIGCHLD and SIGURG.

sigaction is the POSIX way but is complicated, since one argument is a structure that must be allocated and filled in. The signal function is easier, but it predates POSIX and different implementations give it different semantics — so the book defines its own signal wrapper on top of sigaction.

Zombies

When a child terminates, the kernel cannot discard everything immediately because the parent may still want the child's termination status. So it keeps a small record — process ID, termination status, resource usage — and that record is the zombie.

A zombie holds no memory and no descriptors, but it does hold a slot in the process table, and that table is finite. A concurrent server that forks a child per client and never reaps them will fill it.

The handler

void
sig_chld(int signo)
{
    pid_t  pid;
    int    stat;

    while ( (pid = waitpid(-1, &stat, WNOHANG)) > 0)
        printf("child %d terminated\n", pid);
    return;
}

/* in main, before the accept loop: */
Signal(SIGCHLD, sig_chld);

wait versus waitpid

#include <sys/wait.h>

pid_t wait(int *statloc);
pid_t waitpid(pid_t pid, int *statloc, int options);
             Both return: process ID if OK, 0 or -1 on error

Both return two values: the process ID of the terminated child as the return value, and the termination status through statloc. Macros such as WIFEXITED and WEXITSTATUS examine that status.

Why the handler must loop — the key insight

And then accept returns EINTR

Once SIGCHLD is caught, the parent's blocked accept can be interrupted by the signal and return −1 with errno set to EINTR. A slow system call is one that can block forever — accept, read on a socket, connect — and any of them may be interrupted. The server must handle it:

if ( (connfd = accept(listenfd, (SA *) &cliaddr, &clilen)) < 0) {
    if (errno == EINTR)
        continue;              /* restart the accept */
    else
        err_sys("accept error");
}

connect is the exception that must not be restarted: if it is interrupted you must use select to wait for the connection to complete.

1 · Crashing of the server host

1. When the server host crashes, nothing is sent out on the existing network connections — we are assuming a crash, not an orderly shutdown.
2. We type a line to the client. It is written by writen and sent as a data segment; the client then blocks in readline waiting for the echo.
3. Watching with tcpdump you would see the client TCP continually retransmitting the data segment, trying to receive an ACK. Berkeley-derived implementations retransmit 12 times, waiting around 9 minutes before giving up.
4. When the client TCP gives up, an error is returned to the client process. Since it is blocked in readline, that call returns the error: ETIMEDOUT if there were no responses at all, or EHOSTUNREACH / ENETUNREACH if some intermediate router determined the host was unreachable and responded with an ICMP destination-unreachable message.

Note the client only detects this because it sent data. To detect a crashed peer while idle you need SO_KEEPALIVE; to detect it faster than nine minutes, place a timeout on the readline.

2 · Crashing and rebooting of the server host

1. We start the server and then the client, and type a line to verify that the connection is established.
2. The server host crashes and reboots.
3. We type a line of input to the client, which is sent as a TCP data segment to the server host.
4. When the server host reboots after crashing, its TCP loses all information about connections that existed before the crash. Therefore the server TCP responds to the received data segment with an RST.
5. Our client is blocked in the call to readline when the RST is received, causing readline to return the error ECONNRESET.

This is the third of the three conditions that generate an RST: TCP receives a segment for a connection that does not exist. (The other two: a SYN arrives for a port with no listening server, and TCP wants to abort an existing connection.)

3 · Shutdown of the server host

1. When a Unix system is shut down, the init process normally sends SIGTERM to all processes. We can catch this signal.
2. It then waits some fixed amount of time, often between 5 and 20 seconds, giving all running processes a short time to clean up and terminate.
3. It then sends SIGKILL — which we cannot catch — to any processes still running.
4. If we do not catch SIGTERM and terminate, our server is terminated by SIGKILL. Either way, when the process terminates all open descriptors are closed, so a FIN is sent and we follow the same sequence of steps as a normal termination.

Why it takes two writes

A frequently asked question is how to obtain the signal on the first write rather than the second. This is not possible. The first write elicits the RST; the second write elicits the signal.

Note the underlying asymmetry: it is fine to write to a socket that has received a FIN, but it is an error to write to a socket that has received an RST. A FIN only means the peer has finished sending — it may still be reading.

How to handle it

The recommended way depends on what the application wants to do:

• If there is nothing special to do, set the disposition to SIG_IGN, assuming that subsequent output operations will catch the EPIPE error and terminate.
• If special action is needed — writing to a log file, say — catch the signal and perform the action in the handler.

#include <sys/socket.h>

int getsockopt(int sockfd, int level, int optname,
               void *optval, socklen_t *optlen);

int setsockopt(int sockfd, int level, int optname,
               const void *optval, socklen_t optlen);
                                Both return: 0 if OK, -1 on error

The parameters

• sockfd — must refer to an open socket descriptor.
• level — specifies the code in the system that interprets the option: the general socket code or some protocol-specific code (IPv4, IPv6, TCP or SCTP).
• optval — a pointer to a variable from which the new value of the option is fetched by setsockopt, or into which the current value is stored by getsockopt.
• optlen — the size of that variable, as a value for setsockopt and as a value-result for getsockopt.

The two categories

• For a flag option, *optval is an integer. getsockopt returns zero if the option is disabled, non-zero if enabled. setsockopt requires a non-zero *optval to turn the option on and a zero value to turn it off.
• If the "Flag" column does not contain a •, the option is used to pass a value of the specified datatype between the user process and the system.

The book's tables write a datatype with two braces to indicate a structure — linger{} means struct linger.

Socket states and inheritance

Some options have timing considerations. These eleven options are inherited by a connected TCP socket: SO_DEBUG, SO_DONTROUTE, SO_KEEPALIVE, SO_LINGER, SO_OOBINLINE, SO_RCVBUF, SO_RCVLOWAT, SO_SNDBUF, SO_SNDLOWAT, TCP_MAXSEG and TCP_NODELAY.

Two other ways to affect a socket

The fcntl function and the ioctl function.

The four uses of SO_REUSEADDR

1. It allows a listening server to bind its well-known port even if a previously existing connection using that port exists in TIME_WAIT. This is by far the most common use.
2. It allows multiple servers on the same port with different local addresses — common on a multihomed host.
3. It allows a single process to bind the same port to multiple sockets, as long as each binds a different local IP address.
4. It allows a completely duplicate binding — same address and port. Normally supported only for UDP sockets, typically for multicast.

Why the restart fails without it

When the server is killed, its socket goes into TIME_WAIT for 2MSL, and TCP will not create a new incarnation of a socket pair that is currently in TIME_WAIT. The server has no choice about its port — it must bind the well-known one — so bind fails with EADDRINUSE for up to four minutes.

A client never has this problem because it binds an ephemeral port, so on restart it is simply given a different one.

The code — note where it goes

int  on = 1;

sockfd = Socket(AF_INET, SOCK_STREAM, 0);

Setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));

Bind(sockfd, (SA *) &servaddr, sizeof(servaddr));

Between socket and bind — after the bind it is useless. The rule of thumb: all TCP servers should specify SO_REUSEADDR to allow the server to be restarted.

Why SO_LINGER is the wrong fix

SO_REUSEPORT, for completeness

Allows a completely duplicate binding — same address and same port — but only if every socket that wants the binding sets the option, including the first. Normally supported only for UDP, and only when the address being bound is a multicast address.

SO_KEEPALIVE

When set on a TCP socket, if no data has been exchanged in either direction for two hours, TCP sends a keep-alive probe to the peer. Three outcomes:

This is the answer to the problem Chapter 5 leaves open: how an idle client detects that the server host has crashed.

SO_RCVBUF and SO_SNDBUF

Every socket has a send buffer and a receive buffer; these change their sizes.

Bandwidth-delay product

The capacity of the pipe = bandwidth (bytes/second) × round-trip time (seconds). The TCP send buffer must hold data until it is acknowledged, so if it is smaller than this product the connection cannot keep the pipe full and throughput is limited by the buffer rather than the link.

Worked: a 100 Mbit/s link with a 60 ms RTT. 100,000,000 ÷ 8 = 12,500,000 bytes/s; × 0.06 s = 750,000 bytes in flight. With a default 64 KB buffer you would use under a tenth of the link — which is exactly why the window scale option had to be invented, since without it the advertised window cannot exceed 65,535.

TCP_NODELAY and the Nagle algorithm

The Nagle algorithm reduces the number of small packets on a WAN: if a connection has outstanding unacknowledged data, small amounts of new data are held back until the acknowledgment arrives, so at most one small unacknowledged segment is in flight at a time. TCP_NODELAY disables it.

It is disabled because Nagle interacts badly with delayed ACKs: a client that writes a header and a body as two separate writes stalls for the delayed-ACK interval between them on every request. The better fix is usually to write once.

#include	"unp.h"

int
main(int argc, char **argv)
{
    struct sockaddr_in  addr;
    char                str[INET_ADDRSTRLEN];
    uint32_t            net;

    if (argc != 2)
        err_quit("usage: convert <dotted-decimal-address>");

    /* presentation -> numeric */
    if (inet_pton(AF_INET, argv[1], &addr.sin_addr) <= 0)
        err_quit("inet_pton error for %s", argv[1]);

    net = ntohl(addr.sin_addr.s_addr);      /* to host order, to print it */
    printf("presentation : %s\n", argv[1]);
    printf("numeric (hex): 0x%08x\n", net);
    printf("bytes on wire: %u.%u.%u.%u\n",
           (net >> 24) & 0xff, (net >> 16) & 0xff,
           (net >>  8) & 0xff,  net        & 0xff);

    /* numeric -> presentation */
    if (inet_ntop(AF_INET, &addr.sin_addr, str, sizeof(str)) == NULL)
        err_sys("inet_ntop error");

    printf("back again   : %s\n", str);

    exit(0);
}

The four points to explain

• p and n stand for presentation and numeric. Presentation is the text a human types; numeric is the binary value stored in the socket address structure.
• The test is <= 0, not < 0. inet_pton returns 1 on success, 0 if the input was not a valid presentation-format address, and −1 on error — so two of its three returns are failures.
• INET_ADDRSTRLEN is 16 (and INET6_ADDRSTRLEN is 46). Passing the size to inet_ntop is what makes it impossible to overflow the buffer; if len is too small it fails with ENOSPC.
• ntohl is needed before printing. s_addr is stored in network byte order; printing it as a host integer without converting would give a byte-reversed number on a little-endian machine.

If asked for IPv6, change AF_INET to AF_INET6, sockaddr_in to sockaddr_in6, the field to sin6_addr, and the buffer size to INET6_ADDRSTRLEN. Nothing else changes — which is exactly the point of these two functions replacing the three older ones.

Origins

Unix began as UNICS — UNiplexed Information Computing System — single-user and single-process. It was later termed UNIX, a multi-user, multiprocessing operating system.

POSIX

POSIX stands for Portable Operating System Interface, a family of standards developed by the IEEE. Its primary purpose is to maintain compatibility between operating systems, particularly those based on Unix. Standardisation work is done by the Austin Common Standards Revision Group (CSRG), and its output carries both the IEEE POSIX designation and The Open Group's Technical Standard designation.

POSIX.1g — the one this course cares about

The Open Group

An international consortium of vendors and end-user customers from industry, government and academia. Its unification with the IEEE work produced The Single Unix Specification. Most Unix systems today conform to some version of POSIX.1 and POSIX.2, and many comply with Single Unix Specification Version 3.

IETF

The Internet Engineering Task Force — a large, open, international community of network designers, operators, vendors and researchers concerned with the evolution of the Internet architecture and the smooth operation of the Internet. It publishes RFCs — Request For Comments. UDP is RFC 768.