Merge branch 'master' of git://1984.lsi.us.es/nf-next
[~shefty/rdma-dev.git] / net / netfilter / ipvs / ip_vs_core.c
1 /*
2  * IPVS         An implementation of the IP virtual server support for the
3  *              LINUX operating system.  IPVS is now implemented as a module
4  *              over the Netfilter framework. IPVS can be used to build a
5  *              high-performance and highly available server based on a
6  *              cluster of servers.
7  *
8  * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
9  *              Peter Kese <peter.kese@ijs.si>
10  *              Julian Anastasov <ja@ssi.bg>
11  *
12  *              This program is free software; you can redistribute it and/or
13  *              modify it under the terms of the GNU General Public License
14  *              as published by the Free Software Foundation; either version
15  *              2 of the License, or (at your option) any later version.
16  *
17  * The IPVS code for kernel 2.2 was done by Wensong Zhang and Peter Kese,
18  * with changes/fixes from Julian Anastasov, Lars Marowsky-Bree, Horms
19  * and others.
20  *
21  * Changes:
22  *      Paul `Rusty' Russell            properly handle non-linear skbs
23  *      Harald Welte                    don't use nfcache
24  *
25  */
26
27 #define KMSG_COMPONENT "IPVS"
28 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
29
30 #include <linux/module.h>
31 #include <linux/kernel.h>
32 #include <linux/ip.h>
33 #include <linux/tcp.h>
34 #include <linux/sctp.h>
35 #include <linux/icmp.h>
36 #include <linux/slab.h>
37
38 #include <net/ip.h>
39 #include <net/tcp.h>
40 #include <net/udp.h>
41 #include <net/icmp.h>                   /* for icmp_send */
42 #include <net/route.h>
43 #include <net/ip6_checksum.h>
44 #include <net/netns/generic.h>          /* net_generic() */
45
46 #include <linux/netfilter.h>
47 #include <linux/netfilter_ipv4.h>
48
49 #ifdef CONFIG_IP_VS_IPV6
50 #include <net/ipv6.h>
51 #include <linux/netfilter_ipv6.h>
52 #include <net/ip6_route.h>
53 #endif
54
55 #include <net/ip_vs.h>
56
57
58 EXPORT_SYMBOL(register_ip_vs_scheduler);
59 EXPORT_SYMBOL(unregister_ip_vs_scheduler);
60 EXPORT_SYMBOL(ip_vs_proto_name);
61 EXPORT_SYMBOL(ip_vs_conn_new);
62 EXPORT_SYMBOL(ip_vs_conn_in_get);
63 EXPORT_SYMBOL(ip_vs_conn_out_get);
64 #ifdef CONFIG_IP_VS_PROTO_TCP
65 EXPORT_SYMBOL(ip_vs_tcp_conn_listen);
66 #endif
67 EXPORT_SYMBOL(ip_vs_conn_put);
68 #ifdef CONFIG_IP_VS_DEBUG
69 EXPORT_SYMBOL(ip_vs_get_debug_level);
70 #endif
71
72 static int ip_vs_net_id __read_mostly;
73 /* netns cnt used for uniqueness */
74 static atomic_t ipvs_netns_cnt = ATOMIC_INIT(0);
75
76 /* ID used in ICMP lookups */
77 #define icmp_id(icmph)          (((icmph)->un).echo.id)
78 #define icmpv6_id(icmph)        (icmph->icmp6_dataun.u_echo.identifier)
79
80 const char *ip_vs_proto_name(unsigned int proto)
81 {
82         static char buf[20];
83
84         switch (proto) {
85         case IPPROTO_IP:
86                 return "IP";
87         case IPPROTO_UDP:
88                 return "UDP";
89         case IPPROTO_TCP:
90                 return "TCP";
91         case IPPROTO_SCTP:
92                 return "SCTP";
93         case IPPROTO_ICMP:
94                 return "ICMP";
95 #ifdef CONFIG_IP_VS_IPV6
96         case IPPROTO_ICMPV6:
97                 return "ICMPv6";
98 #endif
99         default:
100                 sprintf(buf, "IP_%d", proto);
101                 return buf;
102         }
103 }
104
105 void ip_vs_init_hash_table(struct list_head *table, int rows)
106 {
107         while (--rows >= 0)
108                 INIT_LIST_HEAD(&table[rows]);
109 }
110
111 static inline void
112 ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
113 {
114         struct ip_vs_dest *dest = cp->dest;
115         struct netns_ipvs *ipvs = net_ipvs(skb_net(skb));
116
117         if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
118                 struct ip_vs_cpu_stats *s;
119
120                 s = this_cpu_ptr(dest->stats.cpustats);
121                 s->ustats.inpkts++;
122                 u64_stats_update_begin(&s->syncp);
123                 s->ustats.inbytes += skb->len;
124                 u64_stats_update_end(&s->syncp);
125
126                 s = this_cpu_ptr(dest->svc->stats.cpustats);
127                 s->ustats.inpkts++;
128                 u64_stats_update_begin(&s->syncp);
129                 s->ustats.inbytes += skb->len;
130                 u64_stats_update_end(&s->syncp);
131
132                 s = this_cpu_ptr(ipvs->tot_stats.cpustats);
133                 s->ustats.inpkts++;
134                 u64_stats_update_begin(&s->syncp);
135                 s->ustats.inbytes += skb->len;
136                 u64_stats_update_end(&s->syncp);
137         }
138 }
139
140
141 static inline void
142 ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
143 {
144         struct ip_vs_dest *dest = cp->dest;
145         struct netns_ipvs *ipvs = net_ipvs(skb_net(skb));
146
147         if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
148                 struct ip_vs_cpu_stats *s;
149
150                 s = this_cpu_ptr(dest->stats.cpustats);
151                 s->ustats.outpkts++;
152                 u64_stats_update_begin(&s->syncp);
153                 s->ustats.outbytes += skb->len;
154                 u64_stats_update_end(&s->syncp);
155
156                 s = this_cpu_ptr(dest->svc->stats.cpustats);
157                 s->ustats.outpkts++;
158                 u64_stats_update_begin(&s->syncp);
159                 s->ustats.outbytes += skb->len;
160                 u64_stats_update_end(&s->syncp);
161
162                 s = this_cpu_ptr(ipvs->tot_stats.cpustats);
163                 s->ustats.outpkts++;
164                 u64_stats_update_begin(&s->syncp);
165                 s->ustats.outbytes += skb->len;
166                 u64_stats_update_end(&s->syncp);
167         }
168 }
169
170
171 static inline void
172 ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc)
173 {
174         struct netns_ipvs *ipvs = net_ipvs(svc->net);
175         struct ip_vs_cpu_stats *s;
176
177         s = this_cpu_ptr(cp->dest->stats.cpustats);
178         s->ustats.conns++;
179
180         s = this_cpu_ptr(svc->stats.cpustats);
181         s->ustats.conns++;
182
183         s = this_cpu_ptr(ipvs->tot_stats.cpustats);
184         s->ustats.conns++;
185 }
186
187
188 static inline void
189 ip_vs_set_state(struct ip_vs_conn *cp, int direction,
190                 const struct sk_buff *skb,
191                 struct ip_vs_proto_data *pd)
192 {
193         if (likely(pd->pp->state_transition))
194                 pd->pp->state_transition(cp, direction, skb, pd);
195 }
196
197 static inline int
198 ip_vs_conn_fill_param_persist(const struct ip_vs_service *svc,
199                               struct sk_buff *skb, int protocol,
200                               const union nf_inet_addr *caddr, __be16 cport,
201                               const union nf_inet_addr *vaddr, __be16 vport,
202                               struct ip_vs_conn_param *p)
203 {
204         ip_vs_conn_fill_param(svc->net, svc->af, protocol, caddr, cport, vaddr,
205                               vport, p);
206         p->pe = svc->pe;
207         if (p->pe && p->pe->fill_param)
208                 return p->pe->fill_param(p, skb);
209
210         return 0;
211 }
212
213 /*
214  *  IPVS persistent scheduling function
215  *  It creates a connection entry according to its template if exists,
216  *  or selects a server and creates a connection entry plus a template.
217  *  Locking: we are svc user (svc->refcnt), so we hold all dests too
218  *  Protocols supported: TCP, UDP
219  */
220 static struct ip_vs_conn *
221 ip_vs_sched_persist(struct ip_vs_service *svc,
222                     struct sk_buff *skb, __be16 src_port, __be16 dst_port,
223                     int *ignored, struct ip_vs_iphdr *iph)
224 {
225         struct ip_vs_conn *cp = NULL;
226         struct ip_vs_dest *dest;
227         struct ip_vs_conn *ct;
228         __be16 dport = 0;               /* destination port to forward */
229         unsigned int flags;
230         struct ip_vs_conn_param param;
231         const union nf_inet_addr fwmark = { .ip = htonl(svc->fwmark) };
232         union nf_inet_addr snet;        /* source network of the client,
233                                            after masking */
234
235         /* Mask saddr with the netmask to adjust template granularity */
236 #ifdef CONFIG_IP_VS_IPV6
237         if (svc->af == AF_INET6)
238                 ipv6_addr_prefix(&snet.in6, &iph->saddr.in6, svc->netmask);
239         else
240 #endif
241                 snet.ip = iph->saddr.ip & svc->netmask;
242
243         IP_VS_DBG_BUF(6, "p-schedule: src %s:%u dest %s:%u "
244                       "mnet %s\n",
245                       IP_VS_DBG_ADDR(svc->af, &iph->saddr), ntohs(src_port),
246                       IP_VS_DBG_ADDR(svc->af, &iph->daddr), ntohs(dst_port),
247                       IP_VS_DBG_ADDR(svc->af, &snet));
248
249         /*
250          * As far as we know, FTP is a very complicated network protocol, and
251          * it uses control connection and data connections. For active FTP,
252          * FTP server initialize data connection to the client, its source port
253          * is often 20. For passive FTP, FTP server tells the clients the port
254          * that it passively listens to,  and the client issues the data
255          * connection. In the tunneling or direct routing mode, the load
256          * balancer is on the client-to-server half of connection, the port
257          * number is unknown to the load balancer. So, a conn template like
258          * <caddr, 0, vaddr, 0, daddr, 0> is created for persistent FTP
259          * service, and a template like <caddr, 0, vaddr, vport, daddr, dport>
260          * is created for other persistent services.
261          */
262         {
263                 int protocol = iph->protocol;
264                 const union nf_inet_addr *vaddr = &iph->daddr;
265                 __be16 vport = 0;
266
267                 if (dst_port == svc->port) {
268                         /* non-FTP template:
269                          * <protocol, caddr, 0, vaddr, vport, daddr, dport>
270                          * FTP template:
271                          * <protocol, caddr, 0, vaddr, 0, daddr, 0>
272                          */
273                         if (svc->port != FTPPORT)
274                                 vport = dst_port;
275                 } else {
276                         /* Note: persistent fwmark-based services and
277                          * persistent port zero service are handled here.
278                          * fwmark template:
279                          * <IPPROTO_IP,caddr,0,fwmark,0,daddr,0>
280                          * port zero template:
281                          * <protocol,caddr,0,vaddr,0,daddr,0>
282                          */
283                         if (svc->fwmark) {
284                                 protocol = IPPROTO_IP;
285                                 vaddr = &fwmark;
286                         }
287                 }
288                 /* return *ignored = -1 so NF_DROP can be used */
289                 if (ip_vs_conn_fill_param_persist(svc, skb, protocol, &snet, 0,
290                                                   vaddr, vport, &param) < 0) {
291                         *ignored = -1;
292                         return NULL;
293                 }
294         }
295
296         /* Check if a template already exists */
297         ct = ip_vs_ct_in_get(&param);
298         if (!ct || !ip_vs_check_template(ct)) {
299                 /*
300                  * No template found or the dest of the connection
301                  * template is not available.
302                  * return *ignored=0 i.e. ICMP and NF_DROP
303                  */
304                 dest = svc->scheduler->schedule(svc, skb);
305                 if (!dest) {
306                         IP_VS_DBG(1, "p-schedule: no dest found.\n");
307                         kfree(param.pe_data);
308                         *ignored = 0;
309                         return NULL;
310                 }
311
312                 if (dst_port == svc->port && svc->port != FTPPORT)
313                         dport = dest->port;
314
315                 /* Create a template
316                  * This adds param.pe_data to the template,
317                  * and thus param.pe_data will be destroyed
318                  * when the template expires */
319                 ct = ip_vs_conn_new(&param, &dest->addr, dport,
320                                     IP_VS_CONN_F_TEMPLATE, dest, skb->mark);
321                 if (ct == NULL) {
322                         kfree(param.pe_data);
323                         *ignored = -1;
324                         return NULL;
325                 }
326
327                 ct->timeout = svc->timeout;
328         } else {
329                 /* set destination with the found template */
330                 dest = ct->dest;
331                 kfree(param.pe_data);
332         }
333
334         dport = dst_port;
335         if (dport == svc->port && dest->port)
336                 dport = dest->port;
337
338         flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
339                  && iph->protocol == IPPROTO_UDP) ?
340                 IP_VS_CONN_F_ONE_PACKET : 0;
341
342         /*
343          *    Create a new connection according to the template
344          */
345         ip_vs_conn_fill_param(svc->net, svc->af, iph->protocol, &iph->saddr,
346                               src_port, &iph->daddr, dst_port, &param);
347
348         cp = ip_vs_conn_new(&param, &dest->addr, dport, flags, dest, skb->mark);
349         if (cp == NULL) {
350                 ip_vs_conn_put(ct);
351                 *ignored = -1;
352                 return NULL;
353         }
354
355         /*
356          *    Add its control
357          */
358         ip_vs_control_add(cp, ct);
359         ip_vs_conn_put(ct);
360
361         ip_vs_conn_stats(cp, svc);
362         return cp;
363 }
364
365
366 /*
367  *  IPVS main scheduling function
368  *  It selects a server according to the virtual service, and
369  *  creates a connection entry.
370  *  Protocols supported: TCP, UDP
371  *
372  *  Usage of *ignored
373  *
374  * 1 :   protocol tried to schedule (eg. on SYN), found svc but the
375  *       svc/scheduler decides that this packet should be accepted with
376  *       NF_ACCEPT because it must not be scheduled.
377  *
378  * 0 :   scheduler can not find destination, so try bypass or
379  *       return ICMP and then NF_DROP (ip_vs_leave).
380  *
381  * -1 :  scheduler tried to schedule but fatal error occurred, eg.
382  *       ip_vs_conn_new failure (ENOMEM) or ip_vs_sip_fill_param
383  *       failure such as missing Call-ID, ENOMEM on skb_linearize
384  *       or pe_data. In this case we should return NF_DROP without
385  *       any attempts to send ICMP with ip_vs_leave.
386  */
387 struct ip_vs_conn *
388 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb,
389                struct ip_vs_proto_data *pd, int *ignored,
390                struct ip_vs_iphdr *iph)
391 {
392         struct ip_vs_protocol *pp = pd->pp;
393         struct ip_vs_conn *cp = NULL;
394         struct ip_vs_dest *dest;
395         __be16 _ports[2], *pptr;
396         unsigned int flags;
397
398         *ignored = 1;
399         /*
400          * IPv6 frags, only the first hit here.
401          */
402         pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports, iph);
403         if (pptr == NULL)
404                 return NULL;
405
406         /*
407          * FTPDATA needs this check when using local real server.
408          * Never schedule Active FTPDATA connections from real server.
409          * For LVS-NAT they must be already created. For other methods
410          * with persistence the connection is created on SYN+ACK.
411          */
412         if (pptr[0] == FTPDATA) {
413                 IP_VS_DBG_PKT(12, svc->af, pp, skb, 0,
414                               "Not scheduling FTPDATA");
415                 return NULL;
416         }
417
418         /*
419          *    Do not schedule replies from local real server.
420          */
421         if ((!skb->dev || skb->dev->flags & IFF_LOOPBACK) &&
422             (cp = pp->conn_in_get(svc->af, skb, iph, 1))) {
423                 IP_VS_DBG_PKT(12, svc->af, pp, skb, 0,
424                               "Not scheduling reply for existing connection");
425                 __ip_vs_conn_put(cp);
426                 return NULL;
427         }
428
429         /*
430          *    Persistent service
431          */
432         if (svc->flags & IP_VS_SVC_F_PERSISTENT)
433                 return ip_vs_sched_persist(svc, skb, pptr[0], pptr[1], ignored,
434                                            iph);
435
436         *ignored = 0;
437
438         /*
439          *    Non-persistent service
440          */
441         if (!svc->fwmark && pptr[1] != svc->port) {
442                 if (!svc->port)
443                         pr_err("Schedule: port zero only supported "
444                                "in persistent services, "
445                                "check your ipvs configuration\n");
446                 return NULL;
447         }
448
449         dest = svc->scheduler->schedule(svc, skb);
450         if (dest == NULL) {
451                 IP_VS_DBG(1, "Schedule: no dest found.\n");
452                 return NULL;
453         }
454
455         flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
456                  && iph->protocol == IPPROTO_UDP) ?
457                 IP_VS_CONN_F_ONE_PACKET : 0;
458
459         /*
460          *    Create a connection entry.
461          */
462         {
463                 struct ip_vs_conn_param p;
464
465                 ip_vs_conn_fill_param(svc->net, svc->af, iph->protocol,
466                                       &iph->saddr, pptr[0], &iph->daddr,
467                                       pptr[1], &p);
468                 cp = ip_vs_conn_new(&p, &dest->addr,
469                                     dest->port ? dest->port : pptr[1],
470                                     flags, dest, skb->mark);
471                 if (!cp) {
472                         *ignored = -1;
473                         return NULL;
474                 }
475         }
476
477         IP_VS_DBG_BUF(6, "Schedule fwd:%c c:%s:%u v:%s:%u "
478                       "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
479                       ip_vs_fwd_tag(cp),
480                       IP_VS_DBG_ADDR(svc->af, &cp->caddr), ntohs(cp->cport),
481                       IP_VS_DBG_ADDR(svc->af, &cp->vaddr), ntohs(cp->vport),
482                       IP_VS_DBG_ADDR(svc->af, &cp->daddr), ntohs(cp->dport),
483                       cp->flags, atomic_read(&cp->refcnt));
484
485         ip_vs_conn_stats(cp, svc);
486         return cp;
487 }
488
489
490 /*
491  *  Pass or drop the packet.
492  *  Called by ip_vs_in, when the virtual service is available but
493  *  no destination is available for a new connection.
494  */
495 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
496                 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph)
497 {
498         __be16 _ports[2], *pptr;
499 #ifdef CONFIG_SYSCTL
500         struct net *net;
501         struct netns_ipvs *ipvs;
502         int unicast;
503 #endif
504
505         pptr = frag_safe_skb_hp(skb, iph->len, sizeof(_ports), _ports, iph);
506         if (pptr == NULL) {
507                 ip_vs_service_put(svc);
508                 return NF_DROP;
509         }
510
511 #ifdef CONFIG_SYSCTL
512         net = skb_net(skb);
513
514 #ifdef CONFIG_IP_VS_IPV6
515         if (svc->af == AF_INET6)
516                 unicast = ipv6_addr_type(&iph->daddr.in6) & IPV6_ADDR_UNICAST;
517         else
518 #endif
519                 unicast = (inet_addr_type(net, iph->daddr.ip) == RTN_UNICAST);
520
521         /* if it is fwmark-based service, the cache_bypass sysctl is up
522            and the destination is a non-local unicast, then create
523            a cache_bypass connection entry */
524         ipvs = net_ipvs(net);
525         if (ipvs->sysctl_cache_bypass && svc->fwmark && unicast) {
526                 int ret;
527                 struct ip_vs_conn *cp;
528                 unsigned int flags = (svc->flags & IP_VS_SVC_F_ONEPACKET &&
529                                       iph->protocol == IPPROTO_UDP) ?
530                                       IP_VS_CONN_F_ONE_PACKET : 0;
531                 union nf_inet_addr daddr =  { .all = { 0, 0, 0, 0 } };
532
533                 ip_vs_service_put(svc);
534
535                 /* create a new connection entry */
536                 IP_VS_DBG(6, "%s(): create a cache_bypass entry\n", __func__);
537                 {
538                         struct ip_vs_conn_param p;
539                         ip_vs_conn_fill_param(svc->net, svc->af, iph->protocol,
540                                               &iph->saddr, pptr[0],
541                                               &iph->daddr, pptr[1], &p);
542                         cp = ip_vs_conn_new(&p, &daddr, 0,
543                                             IP_VS_CONN_F_BYPASS | flags,
544                                             NULL, skb->mark);
545                         if (!cp)
546                                 return NF_DROP;
547                 }
548
549                 /* statistics */
550                 ip_vs_in_stats(cp, skb);
551
552                 /* set state */
553                 ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
554
555                 /* transmit the first SYN packet */
556                 ret = cp->packet_xmit(skb, cp, pd->pp, iph);
557                 /* do not touch skb anymore */
558
559                 atomic_inc(&cp->in_pkts);
560                 ip_vs_conn_put(cp);
561                 return ret;
562         }
563 #endif
564
565         /*
566          * When the virtual ftp service is presented, packets destined
567          * for other services on the VIP may get here (except services
568          * listed in the ipvs table), pass the packets, because it is
569          * not ipvs job to decide to drop the packets.
570          */
571         if ((svc->port == FTPPORT) && (pptr[1] != FTPPORT)) {
572                 ip_vs_service_put(svc);
573                 return NF_ACCEPT;
574         }
575
576         ip_vs_service_put(svc);
577
578         /*
579          * Notify the client that the destination is unreachable, and
580          * release the socket buffer.
581          * Since it is in IP layer, the TCP socket is not actually
582          * created, the TCP RST packet cannot be sent, instead that
583          * ICMP_PORT_UNREACH is sent here no matter it is TCP/UDP. --WZ
584          */
585 #ifdef CONFIG_IP_VS_IPV6
586         if (svc->af == AF_INET6) {
587                 if (!skb->dev) {
588                         struct net *net = dev_net(skb_dst(skb)->dev);
589
590                         skb->dev = net->loopback_dev;
591                 }
592                 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
593         } else
594 #endif
595                 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
596
597         return NF_DROP;
598 }
599
600 #ifdef CONFIG_SYSCTL
601
602 static int sysctl_snat_reroute(struct sk_buff *skb)
603 {
604         struct netns_ipvs *ipvs = net_ipvs(skb_net(skb));
605         return ipvs->sysctl_snat_reroute;
606 }
607
608 static int sysctl_nat_icmp_send(struct net *net)
609 {
610         struct netns_ipvs *ipvs = net_ipvs(net);
611         return ipvs->sysctl_nat_icmp_send;
612 }
613
614 static int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs)
615 {
616         return ipvs->sysctl_expire_nodest_conn;
617 }
618
619 #else
620
621 static int sysctl_snat_reroute(struct sk_buff *skb) { return 0; }
622 static int sysctl_nat_icmp_send(struct net *net) { return 0; }
623 static int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) { return 0; }
624
625 #endif
626
627 __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset)
628 {
629         return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0));
630 }
631
632 static inline enum ip_defrag_users ip_vs_defrag_user(unsigned int hooknum)
633 {
634         if (NF_INET_LOCAL_IN == hooknum)
635                 return IP_DEFRAG_VS_IN;
636         if (NF_INET_FORWARD == hooknum)
637                 return IP_DEFRAG_VS_FWD;
638         return IP_DEFRAG_VS_OUT;
639 }
640
641 static inline int ip_vs_gather_frags(struct sk_buff *skb, u_int32_t user)
642 {
643         int err = ip_defrag(skb, user);
644
645         if (!err)
646                 ip_send_check(ip_hdr(skb));
647
648         return err;
649 }
650
651 static int ip_vs_route_me_harder(int af, struct sk_buff *skb)
652 {
653 #ifdef CONFIG_IP_VS_IPV6
654         if (af == AF_INET6) {
655                 if (sysctl_snat_reroute(skb) && ip6_route_me_harder(skb) != 0)
656                         return 1;
657         } else
658 #endif
659                 if ((sysctl_snat_reroute(skb) ||
660                      skb_rtable(skb)->rt_flags & RTCF_LOCAL) &&
661                     ip_route_me_harder(skb, RTN_LOCAL) != 0)
662                         return 1;
663
664         return 0;
665 }
666
667 /*
668  * Packet has been made sufficiently writable in caller
669  * - inout: 1=in->out, 0=out->in
670  */
671 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
672                     struct ip_vs_conn *cp, int inout)
673 {
674         struct iphdr *iph        = ip_hdr(skb);
675         unsigned int icmp_offset = iph->ihl*4;
676         struct icmphdr *icmph    = (struct icmphdr *)(skb_network_header(skb) +
677                                                       icmp_offset);
678         struct iphdr *ciph       = (struct iphdr *)(icmph + 1);
679
680         if (inout) {
681                 iph->saddr = cp->vaddr.ip;
682                 ip_send_check(iph);
683                 ciph->daddr = cp->vaddr.ip;
684                 ip_send_check(ciph);
685         } else {
686                 iph->daddr = cp->daddr.ip;
687                 ip_send_check(iph);
688                 ciph->saddr = cp->daddr.ip;
689                 ip_send_check(ciph);
690         }
691
692         /* the TCP/UDP/SCTP port */
693         if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol ||
694             IPPROTO_SCTP == ciph->protocol) {
695                 __be16 *ports = (void *)ciph + ciph->ihl*4;
696
697                 if (inout)
698                         ports[1] = cp->vport;
699                 else
700                         ports[0] = cp->dport;
701         }
702
703         /* And finally the ICMP checksum */
704         icmph->checksum = 0;
705         icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
706         skb->ip_summed = CHECKSUM_UNNECESSARY;
707
708         if (inout)
709                 IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
710                         "Forwarding altered outgoing ICMP");
711         else
712                 IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph,
713                         "Forwarding altered incoming ICMP");
714 }
715
716 #ifdef CONFIG_IP_VS_IPV6
717 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp,
718                     struct ip_vs_conn *cp, int inout)
719 {
720         struct ipv6hdr *iph      = ipv6_hdr(skb);
721         unsigned int icmp_offset = 0;
722         unsigned int offs        = 0; /* header offset*/
723         int protocol;
724         struct icmp6hdr *icmph;
725         struct ipv6hdr *ciph;
726         unsigned short fragoffs;
727
728         ipv6_find_hdr(skb, &icmp_offset, IPPROTO_ICMPV6, &fragoffs, NULL);
729         icmph = (struct icmp6hdr *)(skb_network_header(skb) + icmp_offset);
730         offs = icmp_offset + sizeof(struct icmp6hdr);
731         ciph = (struct ipv6hdr *)(skb_network_header(skb) + offs);
732
733         protocol = ipv6_find_hdr(skb, &offs, -1, &fragoffs, NULL);
734
735         if (inout) {
736                 iph->saddr = cp->vaddr.in6;
737                 ciph->daddr = cp->vaddr.in6;
738         } else {
739                 iph->daddr = cp->daddr.in6;
740                 ciph->saddr = cp->daddr.in6;
741         }
742
743         /* the TCP/UDP/SCTP port */
744         if (!fragoffs && (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
745                           IPPROTO_SCTP == protocol)) {
746                 __be16 *ports = (void *)(skb_network_header(skb) + offs);
747
748                 IP_VS_DBG(11, "%s() changed port %d to %d\n", __func__,
749                               ntohs(inout ? ports[1] : ports[0]),
750                               ntohs(inout ? cp->vport : cp->dport));
751                 if (inout)
752                         ports[1] = cp->vport;
753                 else
754                         ports[0] = cp->dport;
755         }
756
757         /* And finally the ICMP checksum */
758         icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr,
759                                               skb->len - icmp_offset,
760                                               IPPROTO_ICMPV6, 0);
761         skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset;
762         skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
763         skb->ip_summed = CHECKSUM_PARTIAL;
764
765         if (inout)
766                 IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
767                               (void *)ciph - (void *)iph,
768                               "Forwarding altered outgoing ICMPv6");
769         else
770                 IP_VS_DBG_PKT(11, AF_INET6, pp, skb,
771                               (void *)ciph - (void *)iph,
772                               "Forwarding altered incoming ICMPv6");
773 }
774 #endif
775
776 /* Handle relevant response ICMP messages - forward to the right
777  * destination host.
778  */
779 static int handle_response_icmp(int af, struct sk_buff *skb,
780                                 union nf_inet_addr *snet,
781                                 __u8 protocol, struct ip_vs_conn *cp,
782                                 struct ip_vs_protocol *pp,
783                                 unsigned int offset, unsigned int ihl)
784 {
785         unsigned int verdict = NF_DROP;
786
787         if (IP_VS_FWD_METHOD(cp) != 0) {
788                 pr_err("shouldn't reach here, because the box is on the "
789                        "half connection in the tun/dr module.\n");
790         }
791
792         /* Ensure the checksum is correct */
793         if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
794                 /* Failed checksum! */
795                 IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n",
796                               IP_VS_DBG_ADDR(af, snet));
797                 goto out;
798         }
799
800         if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
801             IPPROTO_SCTP == protocol)
802                 offset += 2 * sizeof(__u16);
803         if (!skb_make_writable(skb, offset))
804                 goto out;
805
806 #ifdef CONFIG_IP_VS_IPV6
807         if (af == AF_INET6)
808                 ip_vs_nat_icmp_v6(skb, pp, cp, 1);
809         else
810 #endif
811                 ip_vs_nat_icmp(skb, pp, cp, 1);
812
813         if (ip_vs_route_me_harder(af, skb))
814                 goto out;
815
816         /* do the statistics and put it back */
817         ip_vs_out_stats(cp, skb);
818
819         skb->ipvs_property = 1;
820         if (!(cp->flags & IP_VS_CONN_F_NFCT))
821                 ip_vs_notrack(skb);
822         else
823                 ip_vs_update_conntrack(skb, cp, 0);
824         verdict = NF_ACCEPT;
825
826 out:
827         __ip_vs_conn_put(cp);
828
829         return verdict;
830 }
831
832 /*
833  *      Handle ICMP messages in the inside-to-outside direction (outgoing).
834  *      Find any that might be relevant, check against existing connections.
835  *      Currently handles error types - unreachable, quench, ttl exceeded.
836  */
837 static int ip_vs_out_icmp(struct sk_buff *skb, int *related,
838                           unsigned int hooknum)
839 {
840         struct iphdr *iph;
841         struct icmphdr  _icmph, *ic;
842         struct iphdr    _ciph, *cih;    /* The ip header contained within the ICMP */
843         struct ip_vs_iphdr ciph;
844         struct ip_vs_conn *cp;
845         struct ip_vs_protocol *pp;
846         unsigned int offset, ihl;
847         union nf_inet_addr snet;
848
849         *related = 1;
850
851         /* reassemble IP fragments */
852         if (ip_is_fragment(ip_hdr(skb))) {
853                 if (ip_vs_gather_frags(skb, ip_vs_defrag_user(hooknum)))
854                         return NF_STOLEN;
855         }
856
857         iph = ip_hdr(skb);
858         offset = ihl = iph->ihl * 4;
859         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
860         if (ic == NULL)
861                 return NF_DROP;
862
863         IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n",
864                   ic->type, ntohs(icmp_id(ic)),
865                   &iph->saddr, &iph->daddr);
866
867         /*
868          * Work through seeing if this is for us.
869          * These checks are supposed to be in an order that means easy
870          * things are checked first to speed up processing.... however
871          * this means that some packets will manage to get a long way
872          * down this stack and then be rejected, but that's life.
873          */
874         if ((ic->type != ICMP_DEST_UNREACH) &&
875             (ic->type != ICMP_SOURCE_QUENCH) &&
876             (ic->type != ICMP_TIME_EXCEEDED)) {
877                 *related = 0;
878                 return NF_ACCEPT;
879         }
880
881         /* Now find the contained IP header */
882         offset += sizeof(_icmph);
883         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
884         if (cih == NULL)
885                 return NF_ACCEPT; /* The packet looks wrong, ignore */
886
887         pp = ip_vs_proto_get(cih->protocol);
888         if (!pp)
889                 return NF_ACCEPT;
890
891         /* Is the embedded protocol header present? */
892         if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
893                      pp->dont_defrag))
894                 return NF_ACCEPT;
895
896         IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
897                       "Checking outgoing ICMP for");
898
899         ip_vs_fill_ip4hdr(cih, &ciph);
900         ciph.len += offset;
901         /* The embedded headers contain source and dest in reverse order */
902         cp = pp->conn_out_get(AF_INET, skb, &ciph, 1);
903         if (!cp)
904                 return NF_ACCEPT;
905
906         snet.ip = iph->saddr;
907         return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp,
908                                     pp, ciph.len, ihl);
909 }
910
911 #ifdef CONFIG_IP_VS_IPV6
912 static int ip_vs_out_icmp_v6(struct sk_buff *skb, int *related,
913                              unsigned int hooknum, struct ip_vs_iphdr *ipvsh)
914 {
915         struct icmp6hdr _icmph, *ic;
916         struct ipv6hdr _ip6h, *ip6h; /* The ip header contained within ICMP */
917         struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};/*Contained IP */
918         struct ip_vs_conn *cp;
919         struct ip_vs_protocol *pp;
920         union nf_inet_addr snet;
921         unsigned int writable;
922
923         *related = 1;
924         ic = frag_safe_skb_hp(skb, ipvsh->len, sizeof(_icmph), &_icmph, ipvsh);
925         if (ic == NULL)
926                 return NF_DROP;
927
928         /*
929          * Work through seeing if this is for us.
930          * These checks are supposed to be in an order that means easy
931          * things are checked first to speed up processing.... however
932          * this means that some packets will manage to get a long way
933          * down this stack and then be rejected, but that's life.
934          */
935         if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
936                 *related = 0;
937                 return NF_ACCEPT;
938         }
939         /* Fragment header that is before ICMP header tells us that:
940          * it's not an error message since they can't be fragmented.
941          */
942         if (ipvsh->flags & IP6_FH_F_FRAG)
943                 return NF_DROP;
944
945         IP_VS_DBG(8, "Outgoing ICMPv6 (%d,%d) %pI6c->%pI6c\n",
946                   ic->icmp6_type, ntohs(icmpv6_id(ic)),
947                   &ipvsh->saddr, &ipvsh->daddr);
948
949         /* Now find the contained IP header */
950         ciph.len = ipvsh->len + sizeof(_icmph);
951         ip6h = skb_header_pointer(skb, ciph.len, sizeof(_ip6h), &_ip6h);
952         if (ip6h == NULL)
953                 return NF_ACCEPT; /* The packet looks wrong, ignore */
954         ciph.saddr.in6 = ip6h->saddr; /* conn_out_get() handles reverse order */
955         ciph.daddr.in6 = ip6h->daddr;
956         /* skip possible IPv6 exthdrs of contained IPv6 packet */
957         ciph.protocol = ipv6_find_hdr(skb, &ciph.len, -1, &ciph.fragoffs, NULL);
958         if (ciph.protocol < 0)
959                 return NF_ACCEPT; /* Contained IPv6 hdr looks wrong, ignore */
960
961         pp = ip_vs_proto_get(ciph.protocol);
962         if (!pp)
963                 return NF_ACCEPT;
964
965         /* The embedded headers contain source and dest in reverse order */
966         cp = pp->conn_out_get(AF_INET6, skb, &ciph, 1);
967         if (!cp)
968                 return NF_ACCEPT;
969
970         snet.in6 = ciph.saddr.in6;
971         writable = ciph.len;
972         return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp,
973                                     pp, writable, sizeof(struct ipv6hdr));
974 }
975 #endif
976
977 /*
978  * Check if sctp chunc is ABORT chunk
979  */
980 static inline int is_sctp_abort(const struct sk_buff *skb, int nh_len)
981 {
982         sctp_chunkhdr_t *sch, schunk;
983         sch = skb_header_pointer(skb, nh_len + sizeof(sctp_sctphdr_t),
984                         sizeof(schunk), &schunk);
985         if (sch == NULL)
986                 return 0;
987         if (sch->type == SCTP_CID_ABORT)
988                 return 1;
989         return 0;
990 }
991
992 static inline int is_tcp_reset(const struct sk_buff *skb, int nh_len)
993 {
994         struct tcphdr _tcph, *th;
995
996         th = skb_header_pointer(skb, nh_len, sizeof(_tcph), &_tcph);
997         if (th == NULL)
998                 return 0;
999         return th->rst;
1000 }
1001
1002 /* Handle response packets: rewrite addresses and send away...
1003  */
1004 static unsigned int
1005 handle_response(int af, struct sk_buff *skb, struct ip_vs_proto_data *pd,
1006                 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)
1007 {
1008         struct ip_vs_protocol *pp = pd->pp;
1009
1010         IP_VS_DBG_PKT(11, af, pp, skb, 0, "Outgoing packet");
1011
1012         if (!skb_make_writable(skb, iph->len))
1013                 goto drop;
1014
1015         /* mangle the packet */
1016         if (pp->snat_handler && !pp->snat_handler(skb, pp, cp, iph))
1017                 goto drop;
1018
1019 #ifdef CONFIG_IP_VS_IPV6
1020         if (af == AF_INET6)
1021                 ipv6_hdr(skb)->saddr = cp->vaddr.in6;
1022         else
1023 #endif
1024         {
1025                 ip_hdr(skb)->saddr = cp->vaddr.ip;
1026                 ip_send_check(ip_hdr(skb));
1027         }
1028
1029         /*
1030          * nf_iterate does not expect change in the skb->dst->dev.
1031          * It looks like it is not fatal to enable this code for hooks
1032          * where our handlers are at the end of the chain list and
1033          * when all next handlers use skb->dst->dev and not outdev.
1034          * It will definitely route properly the inout NAT traffic
1035          * when multiple paths are used.
1036          */
1037
1038         /* For policy routing, packets originating from this
1039          * machine itself may be routed differently to packets
1040          * passing through.  We want this packet to be routed as
1041          * if it came from this machine itself.  So re-compute
1042          * the routing information.
1043          */
1044         if (ip_vs_route_me_harder(af, skb))
1045                 goto drop;
1046
1047         IP_VS_DBG_PKT(10, af, pp, skb, 0, "After SNAT");
1048
1049         ip_vs_out_stats(cp, skb);
1050         ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd);
1051         skb->ipvs_property = 1;
1052         if (!(cp->flags & IP_VS_CONN_F_NFCT))
1053                 ip_vs_notrack(skb);
1054         else
1055                 ip_vs_update_conntrack(skb, cp, 0);
1056         ip_vs_conn_put(cp);
1057
1058         LeaveFunction(11);
1059         return NF_ACCEPT;
1060
1061 drop:
1062         ip_vs_conn_put(cp);
1063         kfree_skb(skb);
1064         LeaveFunction(11);
1065         return NF_STOLEN;
1066 }
1067
1068 /*
1069  *      Check if outgoing packet belongs to the established ip_vs_conn.
1070  */
1071 static unsigned int
1072 ip_vs_out(unsigned int hooknum, struct sk_buff *skb, int af)
1073 {
1074         struct net *net = NULL;
1075         struct ip_vs_iphdr iph;
1076         struct ip_vs_protocol *pp;
1077         struct ip_vs_proto_data *pd;
1078         struct ip_vs_conn *cp;
1079
1080         EnterFunction(11);
1081
1082         /* Already marked as IPVS request or reply? */
1083         if (skb->ipvs_property)
1084                 return NF_ACCEPT;
1085
1086         /* Bad... Do not break raw sockets */
1087         if (unlikely(skb->sk != NULL && hooknum == NF_INET_LOCAL_OUT &&
1088                      af == AF_INET)) {
1089                 struct sock *sk = skb->sk;
1090                 struct inet_sock *inet = inet_sk(skb->sk);
1091
1092                 if (inet && sk->sk_family == PF_INET && inet->nodefrag)
1093                         return NF_ACCEPT;
1094         }
1095
1096         if (unlikely(!skb_dst(skb)))
1097                 return NF_ACCEPT;
1098
1099         net = skb_net(skb);
1100         if (!net_ipvs(net)->enable)
1101                 return NF_ACCEPT;
1102
1103         ip_vs_fill_iph_skb(af, skb, &iph);
1104 #ifdef CONFIG_IP_VS_IPV6
1105         if (af == AF_INET6) {
1106                 if (!iph.fragoffs && skb_nfct_reasm(skb)) {
1107                         struct sk_buff *reasm = skb_nfct_reasm(skb);
1108                         /* Save fw mark for coming frags */
1109                         reasm->ipvs_property = 1;
1110                         reasm->mark = skb->mark;
1111                 }
1112                 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1113                         int related;
1114                         int verdict = ip_vs_out_icmp_v6(skb, &related,
1115                                                         hooknum, &iph);
1116
1117                         if (related)
1118                                 return verdict;
1119                 }
1120         } else
1121 #endif
1122                 if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1123                         int related;
1124                         int verdict = ip_vs_out_icmp(skb, &related, hooknum);
1125
1126                         if (related)
1127                                 return verdict;
1128                 }
1129
1130         pd = ip_vs_proto_data_get(net, iph.protocol);
1131         if (unlikely(!pd))
1132                 return NF_ACCEPT;
1133         pp = pd->pp;
1134
1135         /* reassemble IP fragments */
1136 #ifdef CONFIG_IP_VS_IPV6
1137         if (af == AF_INET)
1138 #endif
1139                 if (unlikely(ip_is_fragment(ip_hdr(skb)) && !pp->dont_defrag)) {
1140                         if (ip_vs_gather_frags(skb,
1141                                                ip_vs_defrag_user(hooknum)))
1142                                 return NF_STOLEN;
1143
1144                         ip_vs_fill_ip4hdr(skb_network_header(skb), &iph);
1145                 }
1146
1147         /*
1148          * Check if the packet belongs to an existing entry
1149          */
1150         cp = pp->conn_out_get(af, skb, &iph, 0);
1151
1152         if (likely(cp))
1153                 return handle_response(af, skb, pd, cp, &iph);
1154         if (sysctl_nat_icmp_send(net) &&
1155             (pp->protocol == IPPROTO_TCP ||
1156              pp->protocol == IPPROTO_UDP ||
1157              pp->protocol == IPPROTO_SCTP)) {
1158                 __be16 _ports[2], *pptr;
1159
1160                 pptr = frag_safe_skb_hp(skb, iph.len,
1161                                          sizeof(_ports), _ports, &iph);
1162                 if (pptr == NULL)
1163                         return NF_ACCEPT;       /* Not for me */
1164                 if (ip_vs_lookup_real_service(net, af, iph.protocol,
1165                                               &iph.saddr,
1166                                               pptr[0])) {
1167                         /*
1168                          * Notify the real server: there is no
1169                          * existing entry if it is not RST
1170                          * packet or not TCP packet.
1171                          */
1172                         if ((iph.protocol != IPPROTO_TCP &&
1173                              iph.protocol != IPPROTO_SCTP)
1174                              || ((iph.protocol == IPPROTO_TCP
1175                                   && !is_tcp_reset(skb, iph.len))
1176                                  || (iph.protocol == IPPROTO_SCTP
1177                                         && !is_sctp_abort(skb,
1178                                                 iph.len)))) {
1179 #ifdef CONFIG_IP_VS_IPV6
1180                                 if (af == AF_INET6) {
1181                                         if (!skb->dev)
1182                                                 skb->dev = net->loopback_dev;
1183                                         icmpv6_send(skb,
1184                                                     ICMPV6_DEST_UNREACH,
1185                                                     ICMPV6_PORT_UNREACH,
1186                                                     0);
1187                                 } else
1188 #endif
1189                                         icmp_send(skb,
1190                                                   ICMP_DEST_UNREACH,
1191                                                   ICMP_PORT_UNREACH, 0);
1192                                 return NF_DROP;
1193                         }
1194                 }
1195         }
1196         IP_VS_DBG_PKT(12, af, pp, skb, 0,
1197                       "ip_vs_out: packet continues traversal as normal");
1198         return NF_ACCEPT;
1199 }
1200
1201 /*
1202  *      It is hooked at the NF_INET_FORWARD and NF_INET_LOCAL_IN chain,
1203  *      used only for VS/NAT.
1204  *      Check if packet is reply for established ip_vs_conn.
1205  */
1206 static unsigned int
1207 ip_vs_reply4(unsigned int hooknum, struct sk_buff *skb,
1208              const struct net_device *in, const struct net_device *out,
1209              int (*okfn)(struct sk_buff *))
1210 {
1211         return ip_vs_out(hooknum, skb, AF_INET);
1212 }
1213
1214 /*
1215  *      It is hooked at the NF_INET_LOCAL_OUT chain, used only for VS/NAT.
1216  *      Check if packet is reply for established ip_vs_conn.
1217  */
1218 static unsigned int
1219 ip_vs_local_reply4(unsigned int hooknum, struct sk_buff *skb,
1220                    const struct net_device *in, const struct net_device *out,
1221                    int (*okfn)(struct sk_buff *))
1222 {
1223         unsigned int verdict;
1224
1225         /* Disable BH in LOCAL_OUT until all places are fixed */
1226         local_bh_disable();
1227         verdict = ip_vs_out(hooknum, skb, AF_INET);
1228         local_bh_enable();
1229         return verdict;
1230 }
1231
1232 #ifdef CONFIG_IP_VS_IPV6
1233
1234 /*
1235  *      It is hooked at the NF_INET_FORWARD and NF_INET_LOCAL_IN chain,
1236  *      used only for VS/NAT.
1237  *      Check if packet is reply for established ip_vs_conn.
1238  */
1239 static unsigned int
1240 ip_vs_reply6(unsigned int hooknum, struct sk_buff *skb,
1241              const struct net_device *in, const struct net_device *out,
1242              int (*okfn)(struct sk_buff *))
1243 {
1244         return ip_vs_out(hooknum, skb, AF_INET6);
1245 }
1246
1247 /*
1248  *      It is hooked at the NF_INET_LOCAL_OUT chain, used only for VS/NAT.
1249  *      Check if packet is reply for established ip_vs_conn.
1250  */
1251 static unsigned int
1252 ip_vs_local_reply6(unsigned int hooknum, struct sk_buff *skb,
1253                    const struct net_device *in, const struct net_device *out,
1254                    int (*okfn)(struct sk_buff *))
1255 {
1256         unsigned int verdict;
1257
1258         /* Disable BH in LOCAL_OUT until all places are fixed */
1259         local_bh_disable();
1260         verdict = ip_vs_out(hooknum, skb, AF_INET6);
1261         local_bh_enable();
1262         return verdict;
1263 }
1264
1265 #endif
1266
1267 /*
1268  *      Handle ICMP messages in the outside-to-inside direction (incoming).
1269  *      Find any that might be relevant, check against existing connections,
1270  *      forward to the right destination host if relevant.
1271  *      Currently handles error types - unreachable, quench, ttl exceeded.
1272  */
1273 static int
1274 ip_vs_in_icmp(struct sk_buff *skb, int *related, unsigned int hooknum)
1275 {
1276         struct net *net = NULL;
1277         struct iphdr *iph;
1278         struct icmphdr  _icmph, *ic;
1279         struct iphdr    _ciph, *cih;    /* The ip header contained within the ICMP */
1280         struct ip_vs_iphdr ciph;
1281         struct ip_vs_conn *cp;
1282         struct ip_vs_protocol *pp;
1283         struct ip_vs_proto_data *pd;
1284         unsigned int offset, offset2, ihl, verdict;
1285         bool ipip;
1286
1287         *related = 1;
1288
1289         /* reassemble IP fragments */
1290         if (ip_is_fragment(ip_hdr(skb))) {
1291                 if (ip_vs_gather_frags(skb, ip_vs_defrag_user(hooknum)))
1292                         return NF_STOLEN;
1293         }
1294
1295         iph = ip_hdr(skb);
1296         offset = ihl = iph->ihl * 4;
1297         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
1298         if (ic == NULL)
1299                 return NF_DROP;
1300
1301         IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n",
1302                   ic->type, ntohs(icmp_id(ic)),
1303                   &iph->saddr, &iph->daddr);
1304
1305         /*
1306          * Work through seeing if this is for us.
1307          * These checks are supposed to be in an order that means easy
1308          * things are checked first to speed up processing.... however
1309          * this means that some packets will manage to get a long way
1310          * down this stack and then be rejected, but that's life.
1311          */
1312         if ((ic->type != ICMP_DEST_UNREACH) &&
1313             (ic->type != ICMP_SOURCE_QUENCH) &&
1314             (ic->type != ICMP_TIME_EXCEEDED)) {
1315                 *related = 0;
1316                 return NF_ACCEPT;
1317         }
1318
1319         /* Now find the contained IP header */
1320         offset += sizeof(_icmph);
1321         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1322         if (cih == NULL)
1323                 return NF_ACCEPT; /* The packet looks wrong, ignore */
1324
1325         net = skb_net(skb);
1326
1327         /* Special case for errors for IPIP packets */
1328         ipip = false;
1329         if (cih->protocol == IPPROTO_IPIP) {
1330                 if (unlikely(cih->frag_off & htons(IP_OFFSET)))
1331                         return NF_ACCEPT;
1332                 /* Error for our IPIP must arrive at LOCAL_IN */
1333                 if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL))
1334                         return NF_ACCEPT;
1335                 offset += cih->ihl * 4;
1336                 cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1337                 if (cih == NULL)
1338                         return NF_ACCEPT; /* The packet looks wrong, ignore */
1339                 ipip = true;
1340         }
1341
1342         pd = ip_vs_proto_data_get(net, cih->protocol);
1343         if (!pd)
1344                 return NF_ACCEPT;
1345         pp = pd->pp;
1346
1347         /* Is the embedded protocol header present? */
1348         if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
1349                      pp->dont_defrag))
1350                 return NF_ACCEPT;
1351
1352         IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset,
1353                       "Checking incoming ICMP for");
1354
1355         offset2 = offset;
1356         ip_vs_fill_ip4hdr(cih, &ciph);
1357         ciph.len += offset;
1358         offset = ciph.len;
1359         /* The embedded headers contain source and dest in reverse order.
1360          * For IPIP this is error for request, not for reply.
1361          */
1362         cp = pp->conn_in_get(AF_INET, skb, &ciph, ipip ? 0 : 1);
1363         if (!cp)
1364                 return NF_ACCEPT;
1365
1366         verdict = NF_DROP;
1367
1368         /* Ensure the checksum is correct */
1369         if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
1370                 /* Failed checksum! */
1371                 IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n",
1372                           &iph->saddr);
1373                 goto out;
1374         }
1375
1376         if (ipip) {
1377                 __be32 info = ic->un.gateway;
1378
1379                 /* Update the MTU */
1380                 if (ic->type == ICMP_DEST_UNREACH &&
1381                     ic->code == ICMP_FRAG_NEEDED) {
1382                         struct ip_vs_dest *dest = cp->dest;
1383                         u32 mtu = ntohs(ic->un.frag.mtu);
1384
1385                         /* Strip outer IP and ICMP, go to IPIP header */
1386                         __skb_pull(skb, ihl + sizeof(_icmph));
1387                         offset2 -= ihl + sizeof(_icmph);
1388                         skb_reset_network_header(skb);
1389                         IP_VS_DBG(12, "ICMP for IPIP %pI4->%pI4: mtu=%u\n",
1390                                 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, mtu);
1391                         ipv4_update_pmtu(skb, dev_net(skb->dev),
1392                                          mtu, 0, 0, 0, 0);
1393                         /* Client uses PMTUD? */
1394                         if (!(cih->frag_off & htons(IP_DF)))
1395                                 goto ignore_ipip;
1396                         /* Prefer the resulting PMTU */
1397                         if (dest) {
1398                                 spin_lock(&dest->dst_lock);
1399                                 if (dest->dst_cache)
1400                                         mtu = dst_mtu(dest->dst_cache);
1401                                 spin_unlock(&dest->dst_lock);
1402                         }
1403                         if (mtu > 68 + sizeof(struct iphdr))
1404                                 mtu -= sizeof(struct iphdr);
1405                         info = htonl(mtu);
1406                 }
1407                 /* Strip outer IP, ICMP and IPIP, go to IP header of
1408                  * original request.
1409                  */
1410                 __skb_pull(skb, offset2);
1411                 skb_reset_network_header(skb);
1412                 IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n",
1413                         &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1414                         ic->type, ic->code, ntohl(info));
1415                 icmp_send(skb, ic->type, ic->code, info);
1416                 /* ICMP can be shorter but anyways, account it */
1417                 ip_vs_out_stats(cp, skb);
1418
1419 ignore_ipip:
1420                 consume_skb(skb);
1421                 verdict = NF_STOLEN;
1422                 goto out;
1423         }
1424
1425         /* do the statistics and put it back */
1426         ip_vs_in_stats(cp, skb);
1427         if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol)
1428                 offset += 2 * sizeof(__u16);
1429         verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph);
1430
1431 out:
1432         __ip_vs_conn_put(cp);
1433
1434         return verdict;
1435 }
1436
1437 #ifdef CONFIG_IP_VS_IPV6
1438 static int ip_vs_in_icmp_v6(struct sk_buff *skb, int *related,
1439                             unsigned int hooknum, struct ip_vs_iphdr *iph)
1440 {
1441         struct net *net = NULL;
1442         struct ipv6hdr _ip6h, *ip6h;
1443         struct icmp6hdr _icmph, *ic;
1444         struct ip_vs_iphdr ciph = {.flags = 0, .fragoffs = 0};/*Contained IP */
1445         struct ip_vs_conn *cp;
1446         struct ip_vs_protocol *pp;
1447         struct ip_vs_proto_data *pd;
1448         unsigned int offs_ciph, writable, verdict;
1449
1450         *related = 1;
1451
1452         ic = frag_safe_skb_hp(skb, iph->len, sizeof(_icmph), &_icmph, iph);
1453         if (ic == NULL)
1454                 return NF_DROP;
1455
1456         /*
1457          * Work through seeing if this is for us.
1458          * These checks are supposed to be in an order that means easy
1459          * things are checked first to speed up processing.... however
1460          * this means that some packets will manage to get a long way
1461          * down this stack and then be rejected, but that's life.
1462          */
1463         if (ic->icmp6_type & ICMPV6_INFOMSG_MASK) {
1464                 *related = 0;
1465                 return NF_ACCEPT;
1466         }
1467         /* Fragment header that is before ICMP header tells us that:
1468          * it's not an error message since they can't be fragmented.
1469          */
1470         if (iph->flags & IP6_FH_F_FRAG)
1471                 return NF_DROP;
1472
1473         IP_VS_DBG(8, "Incoming ICMPv6 (%d,%d) %pI6c->%pI6c\n",
1474                   ic->icmp6_type, ntohs(icmpv6_id(ic)),
1475                   &iph->saddr, &iph->daddr);
1476
1477         /* Now find the contained IP header */
1478         ciph.len = iph->len + sizeof(_icmph);
1479         offs_ciph = ciph.len; /* Save ip header offset */
1480         ip6h = skb_header_pointer(skb, ciph.len, sizeof(_ip6h), &_ip6h);
1481         if (ip6h == NULL)
1482                 return NF_ACCEPT; /* The packet looks wrong, ignore */
1483         ciph.saddr.in6 = ip6h->saddr; /* conn_in_get() handles reverse order */
1484         ciph.daddr.in6 = ip6h->daddr;
1485         /* skip possible IPv6 exthdrs of contained IPv6 packet */
1486         ciph.protocol = ipv6_find_hdr(skb, &ciph.len, -1, &ciph.fragoffs, NULL);
1487         if (ciph.protocol < 0)
1488                 return NF_ACCEPT; /* Contained IPv6 hdr looks wrong, ignore */
1489
1490         net = skb_net(skb);
1491         pd = ip_vs_proto_data_get(net, ciph.protocol);
1492         if (!pd)
1493                 return NF_ACCEPT;
1494         pp = pd->pp;
1495
1496         /* Cannot handle fragmented embedded protocol */
1497         if (ciph.fragoffs)
1498                 return NF_ACCEPT;
1499
1500         IP_VS_DBG_PKT(11, AF_INET6, pp, skb, offs_ciph,
1501                       "Checking incoming ICMPv6 for");
1502
1503         /* The embedded headers contain source and dest in reverse order
1504          * if not from localhost
1505          */
1506         cp = pp->conn_in_get(AF_INET6, skb, &ciph,
1507                              (hooknum == NF_INET_LOCAL_OUT) ? 0 : 1);
1508
1509         if (!cp)
1510                 return NF_ACCEPT;
1511         /* VS/TUN, VS/DR and LOCALNODE just let it go */
1512         if ((hooknum == NF_INET_LOCAL_OUT) &&
1513             (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)) {
1514                 __ip_vs_conn_put(cp);
1515                 return NF_ACCEPT;
1516         }
1517
1518         /* do the statistics and put it back */
1519         ip_vs_in_stats(cp, skb);
1520
1521         /* Need to mangle contained IPv6 header in ICMPv6 packet */
1522         writable = ciph.len;
1523         if (IPPROTO_TCP == ciph.protocol || IPPROTO_UDP == ciph.protocol ||
1524             IPPROTO_SCTP == ciph.protocol)
1525                 writable += 2 * sizeof(__u16); /* Also mangle ports */
1526
1527         verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, writable, hooknum, &ciph);
1528
1529         __ip_vs_conn_put(cp);
1530
1531         return verdict;
1532 }
1533 #endif
1534
1535
1536 /*
1537  *      Check if it's for virtual services, look it up,
1538  *      and send it on its way...
1539  */
1540 static unsigned int
1541 ip_vs_in(unsigned int hooknum, struct sk_buff *skb, int af)
1542 {
1543         struct net *net;
1544         struct ip_vs_iphdr iph;
1545         struct ip_vs_protocol *pp;
1546         struct ip_vs_proto_data *pd;
1547         struct ip_vs_conn *cp;
1548         int ret, pkts;
1549         struct netns_ipvs *ipvs;
1550
1551         /* Already marked as IPVS request or reply? */
1552         if (skb->ipvs_property)
1553                 return NF_ACCEPT;
1554
1555         /*
1556          *      Big tappo:
1557          *      - remote client: only PACKET_HOST
1558          *      - route: used for struct net when skb->dev is unset
1559          */
1560         if (unlikely((skb->pkt_type != PACKET_HOST &&
1561                       hooknum != NF_INET_LOCAL_OUT) ||
1562                      !skb_dst(skb))) {
1563                 ip_vs_fill_iph_skb(af, skb, &iph);
1564                 IP_VS_DBG_BUF(12, "packet type=%d proto=%d daddr=%s"
1565                               " ignored in hook %u\n",
1566                               skb->pkt_type, iph.protocol,
1567                               IP_VS_DBG_ADDR(af, &iph.daddr), hooknum);
1568                 return NF_ACCEPT;
1569         }
1570         /* ipvs enabled in this netns ? */
1571         net = skb_net(skb);
1572         ipvs = net_ipvs(net);
1573         if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
1574                 return NF_ACCEPT;
1575
1576         ip_vs_fill_iph_skb(af, skb, &iph);
1577
1578         /* Bad... Do not break raw sockets */
1579         if (unlikely(skb->sk != NULL && hooknum == NF_INET_LOCAL_OUT &&
1580                      af == AF_INET)) {
1581                 struct sock *sk = skb->sk;
1582                 struct inet_sock *inet = inet_sk(skb->sk);
1583
1584                 if (inet && sk->sk_family == PF_INET && inet->nodefrag)
1585                         return NF_ACCEPT;
1586         }
1587
1588 #ifdef CONFIG_IP_VS_IPV6
1589         if (af == AF_INET6) {
1590                 if (!iph.fragoffs && skb_nfct_reasm(skb)) {
1591                         struct sk_buff *reasm = skb_nfct_reasm(skb);
1592                         /* Save fw mark for coming frags. */
1593                         reasm->ipvs_property = 1;
1594                         reasm->mark = skb->mark;
1595                 }
1596                 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1597                         int related;
1598                         int verdict = ip_vs_in_icmp_v6(skb, &related, hooknum,
1599                                                        &iph);
1600
1601                         if (related)
1602                                 return verdict;
1603                 }
1604         } else
1605 #endif
1606                 if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1607                         int related;
1608                         int verdict = ip_vs_in_icmp(skb, &related, hooknum);
1609
1610                         if (related)
1611                                 return verdict;
1612                 }
1613
1614         /* Protocol supported? */
1615         pd = ip_vs_proto_data_get(net, iph.protocol);
1616         if (unlikely(!pd))
1617                 return NF_ACCEPT;
1618         pp = pd->pp;
1619         /*
1620          * Check if the packet belongs to an existing connection entry
1621          */
1622         cp = pp->conn_in_get(af, skb, &iph, 0);
1623         if (unlikely(!cp) && !iph.fragoffs) {
1624                 /* No (second) fragments need to enter here, as nf_defrag_ipv6
1625                  * replayed fragment zero will already have created the cp
1626                  */
1627                 int v;
1628
1629                 /* Schedule and create new connection entry into &cp */
1630                 if (!pp->conn_schedule(af, skb, pd, &v, &cp, &iph))
1631                         return v;
1632         }
1633
1634         if (unlikely(!cp)) {
1635                 /* sorry, all this trouble for a no-hit :) */
1636                 IP_VS_DBG_PKT(12, af, pp, skb, 0,
1637                               "ip_vs_in: packet continues traversal as normal");
1638                 if (iph.fragoffs && !skb_nfct_reasm(skb)) {
1639                         /* Fragment that couldn't be mapped to a conn entry
1640                          * and don't have any pointer to a reasm skb
1641                          * is missing module nf_defrag_ipv6
1642                          */
1643                         IP_VS_DBG_RL("Unhandled frag, load nf_defrag_ipv6\n");
1644                         IP_VS_DBG_PKT(7, af, pp, skb, 0, "unhandled fragment");
1645                 }
1646                 return NF_ACCEPT;
1647         }
1648
1649         IP_VS_DBG_PKT(11, af, pp, skb, 0, "Incoming packet");
1650         /* Check the server status */
1651         if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
1652                 /* the destination server is not available */
1653
1654                 if (sysctl_expire_nodest_conn(ipvs)) {
1655                         /* try to expire the connection immediately */
1656                         ip_vs_conn_expire_now(cp);
1657                 }
1658                 /* don't restart its timer, and silently
1659                    drop the packet. */
1660                 __ip_vs_conn_put(cp);
1661                 return NF_DROP;
1662         }
1663
1664         ip_vs_in_stats(cp, skb);
1665         ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd);
1666         if (cp->packet_xmit)
1667                 ret = cp->packet_xmit(skb, cp, pp, &iph);
1668                 /* do not touch skb anymore */
1669         else {
1670                 IP_VS_DBG_RL("warning: packet_xmit is null");
1671                 ret = NF_ACCEPT;
1672         }
1673
1674         /* Increase its packet counter and check if it is needed
1675          * to be synchronized
1676          *
1677          * Sync connection if it is about to close to
1678          * encorage the standby servers to update the connections timeout
1679          *
1680          * For ONE_PKT let ip_vs_sync_conn() do the filter work.
1681          */
1682
1683         if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
1684                 pkts = sysctl_sync_threshold(ipvs);
1685         else
1686                 pkts = atomic_add_return(1, &cp->in_pkts);
1687
1688         if (ipvs->sync_state & IP_VS_STATE_MASTER)
1689                 ip_vs_sync_conn(net, cp, pkts);
1690
1691         ip_vs_conn_put(cp);
1692         return ret;
1693 }
1694
1695 /*
1696  *      AF_INET handler in NF_INET_LOCAL_IN chain
1697  *      Schedule and forward packets from remote clients
1698  */
1699 static unsigned int
1700 ip_vs_remote_request4(unsigned int hooknum, struct sk_buff *skb,
1701                       const struct net_device *in,
1702                       const struct net_device *out,
1703                       int (*okfn)(struct sk_buff *))
1704 {
1705         return ip_vs_in(hooknum, skb, AF_INET);
1706 }
1707
1708 /*
1709  *      AF_INET handler in NF_INET_LOCAL_OUT chain
1710  *      Schedule and forward packets from local clients
1711  */
1712 static unsigned int
1713 ip_vs_local_request4(unsigned int hooknum, struct sk_buff *skb,
1714                      const struct net_device *in, const struct net_device *out,
1715                      int (*okfn)(struct sk_buff *))
1716 {
1717         unsigned int verdict;
1718
1719         /* Disable BH in LOCAL_OUT until all places are fixed */
1720         local_bh_disable();
1721         verdict = ip_vs_in(hooknum, skb, AF_INET);
1722         local_bh_enable();
1723         return verdict;
1724 }
1725
1726 #ifdef CONFIG_IP_VS_IPV6
1727
1728 /*
1729  * AF_INET6 fragment handling
1730  * Copy info from first fragment, to the rest of them.
1731  */
1732 static unsigned int
1733 ip_vs_preroute_frag6(unsigned int hooknum, struct sk_buff *skb,
1734                      const struct net_device *in,
1735                      const struct net_device *out,
1736                      int (*okfn)(struct sk_buff *))
1737 {
1738         struct sk_buff *reasm = skb_nfct_reasm(skb);
1739         struct net *net;
1740
1741         /* Skip if not a "replay" from nf_ct_frag6_output or first fragment.
1742          * ipvs_property is set when checking first fragment
1743          * in ip_vs_in() and ip_vs_out().
1744          */
1745         if (reasm)
1746                 IP_VS_DBG(2, "Fragment recv prop:%d\n", reasm->ipvs_property);
1747         if (!reasm || !reasm->ipvs_property)
1748                 return NF_ACCEPT;
1749
1750         net = skb_net(skb);
1751         if (!net_ipvs(net)->enable)
1752                 return NF_ACCEPT;
1753
1754         /* Copy stored fw mark, saved in ip_vs_{in,out} */
1755         skb->mark = reasm->mark;
1756
1757         return NF_ACCEPT;
1758 }
1759
1760 /*
1761  *      AF_INET6 handler in NF_INET_LOCAL_IN chain
1762  *      Schedule and forward packets from remote clients
1763  */
1764 static unsigned int
1765 ip_vs_remote_request6(unsigned int hooknum, struct sk_buff *skb,
1766                       const struct net_device *in,
1767                       const struct net_device *out,
1768                       int (*okfn)(struct sk_buff *))
1769 {
1770         return ip_vs_in(hooknum, skb, AF_INET6);
1771 }
1772
1773 /*
1774  *      AF_INET6 handler in NF_INET_LOCAL_OUT chain
1775  *      Schedule and forward packets from local clients
1776  */
1777 static unsigned int
1778 ip_vs_local_request6(unsigned int hooknum, struct sk_buff *skb,
1779                      const struct net_device *in, const struct net_device *out,
1780                      int (*okfn)(struct sk_buff *))
1781 {
1782         unsigned int verdict;
1783
1784         /* Disable BH in LOCAL_OUT until all places are fixed */
1785         local_bh_disable();
1786         verdict = ip_vs_in(hooknum, skb, AF_INET6);
1787         local_bh_enable();
1788         return verdict;
1789 }
1790
1791 #endif
1792
1793
1794 /*
1795  *      It is hooked at the NF_INET_FORWARD chain, in order to catch ICMP
1796  *      related packets destined for 0.0.0.0/0.
1797  *      When fwmark-based virtual service is used, such as transparent
1798  *      cache cluster, TCP packets can be marked and routed to ip_vs_in,
1799  *      but ICMP destined for 0.0.0.0/0 cannot not be easily marked and
1800  *      sent to ip_vs_in_icmp. So, catch them at the NF_INET_FORWARD chain
1801  *      and send them to ip_vs_in_icmp.
1802  */
1803 static unsigned int
1804 ip_vs_forward_icmp(unsigned int hooknum, struct sk_buff *skb,
1805                    const struct net_device *in, const struct net_device *out,
1806                    int (*okfn)(struct sk_buff *))
1807 {
1808         int r;
1809         struct net *net;
1810         struct netns_ipvs *ipvs;
1811
1812         if (ip_hdr(skb)->protocol != IPPROTO_ICMP)
1813                 return NF_ACCEPT;
1814
1815         /* ipvs enabled in this netns ? */
1816         net = skb_net(skb);
1817         ipvs = net_ipvs(net);
1818         if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
1819                 return NF_ACCEPT;
1820
1821         return ip_vs_in_icmp(skb, &r, hooknum);
1822 }
1823
1824 #ifdef CONFIG_IP_VS_IPV6
1825 static unsigned int
1826 ip_vs_forward_icmp_v6(unsigned int hooknum, struct sk_buff *skb,
1827                       const struct net_device *in, const struct net_device *out,
1828                       int (*okfn)(struct sk_buff *))
1829 {
1830         int r;
1831         struct net *net;
1832         struct netns_ipvs *ipvs;
1833         struct ip_vs_iphdr iphdr;
1834
1835         ip_vs_fill_iph_skb(AF_INET6, skb, &iphdr);
1836         if (iphdr.protocol != IPPROTO_ICMPV6)
1837                 return NF_ACCEPT;
1838
1839         /* ipvs enabled in this netns ? */
1840         net = skb_net(skb);
1841         ipvs = net_ipvs(net);
1842         if (unlikely(sysctl_backup_only(ipvs) || !ipvs->enable))
1843                 return NF_ACCEPT;
1844
1845         return ip_vs_in_icmp_v6(skb, &r, hooknum, &iphdr);
1846 }
1847 #endif
1848
1849
1850 static struct nf_hook_ops ip_vs_ops[] __read_mostly = {
1851         /* After packet filtering, change source only for VS/NAT */
1852         {
1853                 .hook           = ip_vs_reply4,
1854                 .owner          = THIS_MODULE,
1855                 .pf             = NFPROTO_IPV4,
1856                 .hooknum        = NF_INET_LOCAL_IN,
1857                 .priority       = NF_IP_PRI_NAT_SRC - 2,
1858         },
1859         /* After packet filtering, forward packet through VS/DR, VS/TUN,
1860          * or VS/NAT(change destination), so that filtering rules can be
1861          * applied to IPVS. */
1862         {
1863                 .hook           = ip_vs_remote_request4,
1864                 .owner          = THIS_MODULE,
1865                 .pf             = NFPROTO_IPV4,
1866                 .hooknum        = NF_INET_LOCAL_IN,
1867                 .priority       = NF_IP_PRI_NAT_SRC - 1,
1868         },
1869         /* Before ip_vs_in, change source only for VS/NAT */
1870         {
1871                 .hook           = ip_vs_local_reply4,
1872                 .owner          = THIS_MODULE,
1873                 .pf             = NFPROTO_IPV4,
1874                 .hooknum        = NF_INET_LOCAL_OUT,
1875                 .priority       = NF_IP_PRI_NAT_DST + 1,
1876         },
1877         /* After mangle, schedule and forward local requests */
1878         {
1879                 .hook           = ip_vs_local_request4,
1880                 .owner          = THIS_MODULE,
1881                 .pf             = NFPROTO_IPV4,
1882                 .hooknum        = NF_INET_LOCAL_OUT,
1883                 .priority       = NF_IP_PRI_NAT_DST + 2,
1884         },
1885         /* After packet filtering (but before ip_vs_out_icmp), catch icmp
1886          * destined for 0.0.0.0/0, which is for incoming IPVS connections */
1887         {
1888                 .hook           = ip_vs_forward_icmp,
1889                 .owner          = THIS_MODULE,
1890                 .pf             = NFPROTO_IPV4,
1891                 .hooknum        = NF_INET_FORWARD,
1892                 .priority       = 99,
1893         },
1894         /* After packet filtering, change source only for VS/NAT */
1895         {
1896                 .hook           = ip_vs_reply4,
1897                 .owner          = THIS_MODULE,
1898                 .pf             = NFPROTO_IPV4,
1899                 .hooknum        = NF_INET_FORWARD,
1900                 .priority       = 100,
1901         },
1902 #ifdef CONFIG_IP_VS_IPV6
1903         /* After mangle & nat fetch 2:nd fragment and following */
1904         {
1905                 .hook           = ip_vs_preroute_frag6,
1906                 .owner          = THIS_MODULE,
1907                 .pf             = NFPROTO_IPV6,
1908                 .hooknum        = NF_INET_PRE_ROUTING,
1909                 .priority       = NF_IP6_PRI_NAT_DST + 1,
1910         },
1911         /* After packet filtering, change source only for VS/NAT */
1912         {
1913                 .hook           = ip_vs_reply6,
1914                 .owner          = THIS_MODULE,
1915                 .pf             = NFPROTO_IPV6,
1916                 .hooknum        = NF_INET_LOCAL_IN,
1917                 .priority       = NF_IP6_PRI_NAT_SRC - 2,
1918         },
1919         /* After packet filtering, forward packet through VS/DR, VS/TUN,
1920          * or VS/NAT(change destination), so that filtering rules can be
1921          * applied to IPVS. */
1922         {
1923                 .hook           = ip_vs_remote_request6,
1924                 .owner          = THIS_MODULE,
1925                 .pf             = NFPROTO_IPV6,
1926                 .hooknum        = NF_INET_LOCAL_IN,
1927                 .priority       = NF_IP6_PRI_NAT_SRC - 1,
1928         },
1929         /* Before ip_vs_in, change source only for VS/NAT */
1930         {
1931                 .hook           = ip_vs_local_reply6,
1932                 .owner          = THIS_MODULE,
1933                 .pf             = NFPROTO_IPV4,
1934                 .hooknum        = NF_INET_LOCAL_OUT,
1935                 .priority       = NF_IP6_PRI_NAT_DST + 1,
1936         },
1937         /* After mangle, schedule and forward local requests */
1938         {
1939                 .hook           = ip_vs_local_request6,
1940                 .owner          = THIS_MODULE,
1941                 .pf             = NFPROTO_IPV6,
1942                 .hooknum        = NF_INET_LOCAL_OUT,
1943                 .priority       = NF_IP6_PRI_NAT_DST + 2,
1944         },
1945         /* After packet filtering (but before ip_vs_out_icmp), catch icmp
1946          * destined for 0.0.0.0/0, which is for incoming IPVS connections */
1947         {
1948                 .hook           = ip_vs_forward_icmp_v6,
1949                 .owner          = THIS_MODULE,
1950                 .pf             = NFPROTO_IPV6,
1951                 .hooknum        = NF_INET_FORWARD,
1952                 .priority       = 99,
1953         },
1954         /* After packet filtering, change source only for VS/NAT */
1955         {
1956                 .hook           = ip_vs_reply6,
1957                 .owner          = THIS_MODULE,
1958                 .pf             = NFPROTO_IPV6,
1959                 .hooknum        = NF_INET_FORWARD,
1960                 .priority       = 100,
1961         },
1962 #endif
1963 };
1964 /*
1965  *      Initialize IP Virtual Server netns mem.
1966  */
1967 static int __net_init __ip_vs_init(struct net *net)
1968 {
1969         struct netns_ipvs *ipvs;
1970
1971         ipvs = net_generic(net, ip_vs_net_id);
1972         if (ipvs == NULL)
1973                 return -ENOMEM;
1974
1975         /* Hold the beast until a service is registerd */
1976         ipvs->enable = 0;
1977         ipvs->net = net;
1978         /* Counters used for creating unique names */
1979         ipvs->gen = atomic_read(&ipvs_netns_cnt);
1980         atomic_inc(&ipvs_netns_cnt);
1981         net->ipvs = ipvs;
1982
1983         if (ip_vs_estimator_net_init(net) < 0)
1984                 goto estimator_fail;
1985
1986         if (ip_vs_control_net_init(net) < 0)
1987                 goto control_fail;
1988
1989         if (ip_vs_protocol_net_init(net) < 0)
1990                 goto protocol_fail;
1991
1992         if (ip_vs_app_net_init(net) < 0)
1993                 goto app_fail;
1994
1995         if (ip_vs_conn_net_init(net) < 0)
1996                 goto conn_fail;
1997
1998         if (ip_vs_sync_net_init(net) < 0)
1999                 goto sync_fail;
2000
2001         printk(KERN_INFO "IPVS: Creating netns size=%zu id=%d\n",
2002                          sizeof(struct netns_ipvs), ipvs->gen);
2003         return 0;
2004 /*
2005  * Error handling
2006  */
2007
2008 sync_fail:
2009         ip_vs_conn_net_cleanup(net);
2010 conn_fail:
2011         ip_vs_app_net_cleanup(net);
2012 app_fail:
2013         ip_vs_protocol_net_cleanup(net);
2014 protocol_fail:
2015         ip_vs_control_net_cleanup(net);
2016 control_fail:
2017         ip_vs_estimator_net_cleanup(net);
2018 estimator_fail:
2019         net->ipvs = NULL;
2020         return -ENOMEM;
2021 }
2022
2023 static void __net_exit __ip_vs_cleanup(struct net *net)
2024 {
2025         ip_vs_service_net_cleanup(net); /* ip_vs_flush() with locks */
2026         ip_vs_conn_net_cleanup(net);
2027         ip_vs_app_net_cleanup(net);
2028         ip_vs_protocol_net_cleanup(net);
2029         ip_vs_control_net_cleanup(net);
2030         ip_vs_estimator_net_cleanup(net);
2031         IP_VS_DBG(2, "ipvs netns %d released\n", net_ipvs(net)->gen);
2032         net->ipvs = NULL;
2033 }
2034
2035 static void __net_exit __ip_vs_dev_cleanup(struct net *net)
2036 {
2037         EnterFunction(2);
2038         net_ipvs(net)->enable = 0;      /* Disable packet reception */
2039         smp_wmb();
2040         ip_vs_sync_net_cleanup(net);
2041         LeaveFunction(2);
2042 }
2043
2044 static struct pernet_operations ipvs_core_ops = {
2045         .init = __ip_vs_init,
2046         .exit = __ip_vs_cleanup,
2047         .id   = &ip_vs_net_id,
2048         .size = sizeof(struct netns_ipvs),
2049 };
2050
2051 static struct pernet_operations ipvs_core_dev_ops = {
2052         .exit = __ip_vs_dev_cleanup,
2053 };
2054
2055 /*
2056  *      Initialize IP Virtual Server
2057  */
2058 static int __init ip_vs_init(void)
2059 {
2060         int ret;
2061
2062         ret = ip_vs_control_init();
2063         if (ret < 0) {
2064                 pr_err("can't setup control.\n");
2065                 goto exit;
2066         }
2067
2068         ip_vs_protocol_init();
2069
2070         ret = ip_vs_conn_init();
2071         if (ret < 0) {
2072                 pr_err("can't setup connection table.\n");
2073                 goto cleanup_protocol;
2074         }
2075
2076         ret = register_pernet_subsys(&ipvs_core_ops);   /* Alloc ip_vs struct */
2077         if (ret < 0)
2078                 goto cleanup_conn;
2079
2080         ret = register_pernet_device(&ipvs_core_dev_ops);
2081         if (ret < 0)
2082                 goto cleanup_sub;
2083
2084         ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
2085         if (ret < 0) {
2086                 pr_err("can't register hooks.\n");
2087                 goto cleanup_dev;
2088         }
2089
2090         ret = ip_vs_register_nl_ioctl();
2091         if (ret < 0) {
2092                 pr_err("can't register netlink/ioctl.\n");
2093                 goto cleanup_hooks;
2094         }
2095
2096         pr_info("ipvs loaded.\n");
2097
2098         return ret;
2099
2100 cleanup_hooks:
2101         nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
2102 cleanup_dev:
2103         unregister_pernet_device(&ipvs_core_dev_ops);
2104 cleanup_sub:
2105         unregister_pernet_subsys(&ipvs_core_ops);
2106 cleanup_conn:
2107         ip_vs_conn_cleanup();
2108 cleanup_protocol:
2109         ip_vs_protocol_cleanup();
2110         ip_vs_control_cleanup();
2111 exit:
2112         return ret;
2113 }
2114
2115 static void __exit ip_vs_cleanup(void)
2116 {
2117         ip_vs_unregister_nl_ioctl();
2118         nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
2119         unregister_pernet_device(&ipvs_core_dev_ops);
2120         unregister_pernet_subsys(&ipvs_core_ops);       /* free ip_vs struct */
2121         ip_vs_conn_cleanup();
2122         ip_vs_protocol_cleanup();
2123         ip_vs_control_cleanup();
2124         pr_info("ipvs unloaded.\n");
2125 }
2126
2127 module_init(ip_vs_init);
2128 module_exit(ip_vs_cleanup);
2129 MODULE_LICENSE("GPL");