stack_user.c 29 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* -*- mode: c; c-basic-offset: 8; -*-
  3. * vim: noexpandtab sw=8 ts=8 sts=0:
  4. *
  5. * stack_user.c
  6. *
  7. * Code which interfaces ocfs2 with fs/dlm and a userspace stack.
  8. *
  9. * Copyright (C) 2007 Oracle. All rights reserved.
  10. */
  11. #include <linux/module.h>
  12. #include <linux/fs.h>
  13. #include <linux/miscdevice.h>
  14. #include <linux/mutex.h>
  15. #include <linux/slab.h>
  16. #include <linux/reboot.h>
  17. #include <linux/sched.h>
  18. #include <linux/uaccess.h>
  19. #include "stackglue.h"
  20. #include <linux/dlm_plock.h>
  21. /*
  22. * The control protocol starts with a handshake. Until the handshake
  23. * is complete, the control device will fail all write(2)s.
  24. *
  25. * The handshake is simple. First, the client reads until EOF. Each line
  26. * of output is a supported protocol tag. All protocol tags are a single
  27. * character followed by a two hex digit version number. Currently the
  28. * only things supported is T01, for "Text-base version 0x01". Next, the
  29. * client writes the version they would like to use, including the newline.
  30. * Thus, the protocol tag is 'T01\n'. If the version tag written is
  31. * unknown, -EINVAL is returned. Once the negotiation is complete, the
  32. * client can start sending messages.
  33. *
  34. * The T01 protocol has three messages. First is the "SETN" message.
  35. * It has the following syntax:
  36. *
  37. * SETN<space><8-char-hex-nodenum><newline>
  38. *
  39. * This is 14 characters.
  40. *
  41. * The "SETN" message must be the first message following the protocol.
  42. * It tells ocfs2_control the local node number.
  43. *
  44. * Next comes the "SETV" message. It has the following syntax:
  45. *
  46. * SETV<space><2-char-hex-major><space><2-char-hex-minor><newline>
  47. *
  48. * This is 11 characters.
  49. *
  50. * The "SETV" message sets the filesystem locking protocol version as
  51. * negotiated by the client. The client negotiates based on the maximum
  52. * version advertised in /sys/fs/ocfs2/max_locking_protocol. The major
  53. * number from the "SETV" message must match
  54. * ocfs2_user_plugin.sp_max_proto.pv_major, and the minor number
  55. * must be less than or equal to ...sp_max_version.pv_minor.
  56. *
  57. * Once this information has been set, mounts will be allowed. From this
  58. * point on, the "DOWN" message can be sent for node down notification.
  59. * It has the following syntax:
  60. *
  61. * DOWN<space><32-char-cap-hex-uuid><space><8-char-hex-nodenum><newline>
  62. *
  63. * eg:
  64. *
  65. * DOWN 632A924FDD844190BDA93C0DF6B94899 00000001\n
  66. *
  67. * This is 47 characters.
  68. */
  69. /*
  70. * Whether or not the client has done the handshake.
  71. * For now, we have just one protocol version.
  72. */
  73. #define OCFS2_CONTROL_PROTO "T01\n"
  74. #define OCFS2_CONTROL_PROTO_LEN 4
  75. /* Handshake states */
  76. #define OCFS2_CONTROL_HANDSHAKE_INVALID (0)
  77. #define OCFS2_CONTROL_HANDSHAKE_READ (1)
  78. #define OCFS2_CONTROL_HANDSHAKE_PROTOCOL (2)
  79. #define OCFS2_CONTROL_HANDSHAKE_VALID (3)
  80. /* Messages */
  81. #define OCFS2_CONTROL_MESSAGE_OP_LEN 4
  82. #define OCFS2_CONTROL_MESSAGE_SETNODE_OP "SETN"
  83. #define OCFS2_CONTROL_MESSAGE_SETNODE_TOTAL_LEN 14
  84. #define OCFS2_CONTROL_MESSAGE_SETVERSION_OP "SETV"
  85. #define OCFS2_CONTROL_MESSAGE_SETVERSION_TOTAL_LEN 11
  86. #define OCFS2_CONTROL_MESSAGE_DOWN_OP "DOWN"
  87. #define OCFS2_CONTROL_MESSAGE_DOWN_TOTAL_LEN 47
  88. #define OCFS2_TEXT_UUID_LEN 32
  89. #define OCFS2_CONTROL_MESSAGE_VERNUM_LEN 2
  90. #define OCFS2_CONTROL_MESSAGE_NODENUM_LEN 8
  91. #define VERSION_LOCK "version_lock"
  92. enum ocfs2_connection_type {
  93. WITH_CONTROLD,
  94. NO_CONTROLD
  95. };
  96. /*
  97. * ocfs2_live_connection is refcounted because the filesystem and
  98. * miscdevice sides can detach in different order. Let's just be safe.
  99. */
  100. struct ocfs2_live_connection {
  101. struct list_head oc_list;
  102. struct ocfs2_cluster_connection *oc_conn;
  103. enum ocfs2_connection_type oc_type;
  104. atomic_t oc_this_node;
  105. int oc_our_slot;
  106. struct dlm_lksb oc_version_lksb;
  107. char oc_lvb[DLM_LVB_LEN];
  108. struct completion oc_sync_wait;
  109. wait_queue_head_t oc_wait;
  110. };
  111. struct ocfs2_control_private {
  112. struct list_head op_list;
  113. int op_state;
  114. int op_this_node;
  115. struct ocfs2_protocol_version op_proto;
  116. };
  117. /* SETN<space><8-char-hex-nodenum><newline> */
  118. struct ocfs2_control_message_setn {
  119. char tag[OCFS2_CONTROL_MESSAGE_OP_LEN];
  120. char space;
  121. char nodestr[OCFS2_CONTROL_MESSAGE_NODENUM_LEN];
  122. char newline;
  123. };
  124. /* SETV<space><2-char-hex-major><space><2-char-hex-minor><newline> */
  125. struct ocfs2_control_message_setv {
  126. char tag[OCFS2_CONTROL_MESSAGE_OP_LEN];
  127. char space1;
  128. char major[OCFS2_CONTROL_MESSAGE_VERNUM_LEN];
  129. char space2;
  130. char minor[OCFS2_CONTROL_MESSAGE_VERNUM_LEN];
  131. char newline;
  132. };
  133. /* DOWN<space><32-char-cap-hex-uuid><space><8-char-hex-nodenum><newline> */
  134. struct ocfs2_control_message_down {
  135. char tag[OCFS2_CONTROL_MESSAGE_OP_LEN];
  136. char space1;
  137. char uuid[OCFS2_TEXT_UUID_LEN];
  138. char space2;
  139. char nodestr[OCFS2_CONTROL_MESSAGE_NODENUM_LEN];
  140. char newline;
  141. };
  142. union ocfs2_control_message {
  143. char tag[OCFS2_CONTROL_MESSAGE_OP_LEN];
  144. struct ocfs2_control_message_setn u_setn;
  145. struct ocfs2_control_message_setv u_setv;
  146. struct ocfs2_control_message_down u_down;
  147. };
  148. static struct ocfs2_stack_plugin ocfs2_user_plugin;
  149. static atomic_t ocfs2_control_opened;
  150. static int ocfs2_control_this_node = -1;
  151. static struct ocfs2_protocol_version running_proto;
  152. static LIST_HEAD(ocfs2_live_connection_list);
  153. static LIST_HEAD(ocfs2_control_private_list);
  154. static DEFINE_MUTEX(ocfs2_control_lock);
  155. static inline void ocfs2_control_set_handshake_state(struct file *file,
  156. int state)
  157. {
  158. struct ocfs2_control_private *p = file->private_data;
  159. p->op_state = state;
  160. }
  161. static inline int ocfs2_control_get_handshake_state(struct file *file)
  162. {
  163. struct ocfs2_control_private *p = file->private_data;
  164. return p->op_state;
  165. }
  166. static struct ocfs2_live_connection *ocfs2_connection_find(const char *name)
  167. {
  168. size_t len = strlen(name);
  169. struct ocfs2_live_connection *c;
  170. BUG_ON(!mutex_is_locked(&ocfs2_control_lock));
  171. list_for_each_entry(c, &ocfs2_live_connection_list, oc_list) {
  172. if ((c->oc_conn->cc_namelen == len) &&
  173. !strncmp(c->oc_conn->cc_name, name, len))
  174. return c;
  175. }
  176. return NULL;
  177. }
  178. /*
  179. * ocfs2_live_connection structures are created underneath the ocfs2
  180. * mount path. Since the VFS prevents multiple calls to
  181. * fill_super(), we can't get dupes here.
  182. */
  183. static int ocfs2_live_connection_attach(struct ocfs2_cluster_connection *conn,
  184. struct ocfs2_live_connection *c)
  185. {
  186. int rc = 0;
  187. mutex_lock(&ocfs2_control_lock);
  188. c->oc_conn = conn;
  189. if ((c->oc_type == NO_CONTROLD) || atomic_read(&ocfs2_control_opened))
  190. list_add(&c->oc_list, &ocfs2_live_connection_list);
  191. else {
  192. printk(KERN_ERR
  193. "ocfs2: Userspace control daemon is not present\n");
  194. rc = -ESRCH;
  195. }
  196. mutex_unlock(&ocfs2_control_lock);
  197. return rc;
  198. }
  199. /*
  200. * This function disconnects the cluster connection from ocfs2_control.
  201. * Afterwards, userspace can't affect the cluster connection.
  202. */
  203. static void ocfs2_live_connection_drop(struct ocfs2_live_connection *c)
  204. {
  205. mutex_lock(&ocfs2_control_lock);
  206. list_del_init(&c->oc_list);
  207. c->oc_conn = NULL;
  208. mutex_unlock(&ocfs2_control_lock);
  209. kfree(c);
  210. }
  211. static int ocfs2_control_cfu(void *target, size_t target_len,
  212. const char __user *buf, size_t count)
  213. {
  214. /* The T01 expects write(2) calls to have exactly one command */
  215. if ((count != target_len) ||
  216. (count > sizeof(union ocfs2_control_message)))
  217. return -EINVAL;
  218. if (copy_from_user(target, buf, target_len))
  219. return -EFAULT;
  220. return 0;
  221. }
  222. static ssize_t ocfs2_control_validate_protocol(struct file *file,
  223. const char __user *buf,
  224. size_t count)
  225. {
  226. ssize_t ret;
  227. char kbuf[OCFS2_CONTROL_PROTO_LEN];
  228. ret = ocfs2_control_cfu(kbuf, OCFS2_CONTROL_PROTO_LEN,
  229. buf, count);
  230. if (ret)
  231. return ret;
  232. if (strncmp(kbuf, OCFS2_CONTROL_PROTO, OCFS2_CONTROL_PROTO_LEN))
  233. return -EINVAL;
  234. ocfs2_control_set_handshake_state(file,
  235. OCFS2_CONTROL_HANDSHAKE_PROTOCOL);
  236. return count;
  237. }
  238. static void ocfs2_control_send_down(const char *uuid,
  239. int nodenum)
  240. {
  241. struct ocfs2_live_connection *c;
  242. mutex_lock(&ocfs2_control_lock);
  243. c = ocfs2_connection_find(uuid);
  244. if (c) {
  245. BUG_ON(c->oc_conn == NULL);
  246. c->oc_conn->cc_recovery_handler(nodenum,
  247. c->oc_conn->cc_recovery_data);
  248. }
  249. mutex_unlock(&ocfs2_control_lock);
  250. }
  251. /*
  252. * Called whenever configuration elements are sent to /dev/ocfs2_control.
  253. * If all configuration elements are present, try to set the global
  254. * values. If there is a problem, return an error. Skip any missing
  255. * elements, and only bump ocfs2_control_opened when we have all elements
  256. * and are successful.
  257. */
  258. static int ocfs2_control_install_private(struct file *file)
  259. {
  260. int rc = 0;
  261. int set_p = 1;
  262. struct ocfs2_control_private *p = file->private_data;
  263. BUG_ON(p->op_state != OCFS2_CONTROL_HANDSHAKE_PROTOCOL);
  264. mutex_lock(&ocfs2_control_lock);
  265. if (p->op_this_node < 0) {
  266. set_p = 0;
  267. } else if ((ocfs2_control_this_node >= 0) &&
  268. (ocfs2_control_this_node != p->op_this_node)) {
  269. rc = -EINVAL;
  270. goto out_unlock;
  271. }
  272. if (!p->op_proto.pv_major) {
  273. set_p = 0;
  274. } else if (!list_empty(&ocfs2_live_connection_list) &&
  275. ((running_proto.pv_major != p->op_proto.pv_major) ||
  276. (running_proto.pv_minor != p->op_proto.pv_minor))) {
  277. rc = -EINVAL;
  278. goto out_unlock;
  279. }
  280. if (set_p) {
  281. ocfs2_control_this_node = p->op_this_node;
  282. running_proto.pv_major = p->op_proto.pv_major;
  283. running_proto.pv_minor = p->op_proto.pv_minor;
  284. }
  285. out_unlock:
  286. mutex_unlock(&ocfs2_control_lock);
  287. if (!rc && set_p) {
  288. /* We set the global values successfully */
  289. atomic_inc(&ocfs2_control_opened);
  290. ocfs2_control_set_handshake_state(file,
  291. OCFS2_CONTROL_HANDSHAKE_VALID);
  292. }
  293. return rc;
  294. }
  295. static int ocfs2_control_get_this_node(void)
  296. {
  297. int rc;
  298. mutex_lock(&ocfs2_control_lock);
  299. if (ocfs2_control_this_node < 0)
  300. rc = -EINVAL;
  301. else
  302. rc = ocfs2_control_this_node;
  303. mutex_unlock(&ocfs2_control_lock);
  304. return rc;
  305. }
  306. static int ocfs2_control_do_setnode_msg(struct file *file,
  307. struct ocfs2_control_message_setn *msg)
  308. {
  309. long nodenum;
  310. char *ptr = NULL;
  311. struct ocfs2_control_private *p = file->private_data;
  312. if (ocfs2_control_get_handshake_state(file) !=
  313. OCFS2_CONTROL_HANDSHAKE_PROTOCOL)
  314. return -EINVAL;
  315. if (strncmp(msg->tag, OCFS2_CONTROL_MESSAGE_SETNODE_OP,
  316. OCFS2_CONTROL_MESSAGE_OP_LEN))
  317. return -EINVAL;
  318. if ((msg->space != ' ') || (msg->newline != '\n'))
  319. return -EINVAL;
  320. msg->space = msg->newline = '\0';
  321. nodenum = simple_strtol(msg->nodestr, &ptr, 16);
  322. if (!ptr || *ptr)
  323. return -EINVAL;
  324. if ((nodenum == LONG_MIN) || (nodenum == LONG_MAX) ||
  325. (nodenum > INT_MAX) || (nodenum < 0))
  326. return -ERANGE;
  327. p->op_this_node = nodenum;
  328. return ocfs2_control_install_private(file);
  329. }
  330. static int ocfs2_control_do_setversion_msg(struct file *file,
  331. struct ocfs2_control_message_setv *msg)
  332. {
  333. long major, minor;
  334. char *ptr = NULL;
  335. struct ocfs2_control_private *p = file->private_data;
  336. struct ocfs2_protocol_version *max =
  337. &ocfs2_user_plugin.sp_max_proto;
  338. if (ocfs2_control_get_handshake_state(file) !=
  339. OCFS2_CONTROL_HANDSHAKE_PROTOCOL)
  340. return -EINVAL;
  341. if (strncmp(msg->tag, OCFS2_CONTROL_MESSAGE_SETVERSION_OP,
  342. OCFS2_CONTROL_MESSAGE_OP_LEN))
  343. return -EINVAL;
  344. if ((msg->space1 != ' ') || (msg->space2 != ' ') ||
  345. (msg->newline != '\n'))
  346. return -EINVAL;
  347. msg->space1 = msg->space2 = msg->newline = '\0';
  348. major = simple_strtol(msg->major, &ptr, 16);
  349. if (!ptr || *ptr)
  350. return -EINVAL;
  351. minor = simple_strtol(msg->minor, &ptr, 16);
  352. if (!ptr || *ptr)
  353. return -EINVAL;
  354. /*
  355. * The major must be between 1 and 255, inclusive. The minor
  356. * must be between 0 and 255, inclusive. The version passed in
  357. * must be within the maximum version supported by the filesystem.
  358. */
  359. if ((major == LONG_MIN) || (major == LONG_MAX) ||
  360. (major > (u8)-1) || (major < 1))
  361. return -ERANGE;
  362. if ((minor == LONG_MIN) || (minor == LONG_MAX) ||
  363. (minor > (u8)-1) || (minor < 0))
  364. return -ERANGE;
  365. if ((major != max->pv_major) ||
  366. (minor > max->pv_minor))
  367. return -EINVAL;
  368. p->op_proto.pv_major = major;
  369. p->op_proto.pv_minor = minor;
  370. return ocfs2_control_install_private(file);
  371. }
  372. static int ocfs2_control_do_down_msg(struct file *file,
  373. struct ocfs2_control_message_down *msg)
  374. {
  375. long nodenum;
  376. char *p = NULL;
  377. if (ocfs2_control_get_handshake_state(file) !=
  378. OCFS2_CONTROL_HANDSHAKE_VALID)
  379. return -EINVAL;
  380. if (strncmp(msg->tag, OCFS2_CONTROL_MESSAGE_DOWN_OP,
  381. OCFS2_CONTROL_MESSAGE_OP_LEN))
  382. return -EINVAL;
  383. if ((msg->space1 != ' ') || (msg->space2 != ' ') ||
  384. (msg->newline != '\n'))
  385. return -EINVAL;
  386. msg->space1 = msg->space2 = msg->newline = '\0';
  387. nodenum = simple_strtol(msg->nodestr, &p, 16);
  388. if (!p || *p)
  389. return -EINVAL;
  390. if ((nodenum == LONG_MIN) || (nodenum == LONG_MAX) ||
  391. (nodenum > INT_MAX) || (nodenum < 0))
  392. return -ERANGE;
  393. ocfs2_control_send_down(msg->uuid, nodenum);
  394. return 0;
  395. }
  396. static ssize_t ocfs2_control_message(struct file *file,
  397. const char __user *buf,
  398. size_t count)
  399. {
  400. ssize_t ret;
  401. union ocfs2_control_message msg;
  402. /* Try to catch padding issues */
  403. WARN_ON(offsetof(struct ocfs2_control_message_down, uuid) !=
  404. (sizeof(msg.u_down.tag) + sizeof(msg.u_down.space1)));
  405. memset(&msg, 0, sizeof(union ocfs2_control_message));
  406. ret = ocfs2_control_cfu(&msg, count, buf, count);
  407. if (ret)
  408. goto out;
  409. if ((count == OCFS2_CONTROL_MESSAGE_SETNODE_TOTAL_LEN) &&
  410. !strncmp(msg.tag, OCFS2_CONTROL_MESSAGE_SETNODE_OP,
  411. OCFS2_CONTROL_MESSAGE_OP_LEN))
  412. ret = ocfs2_control_do_setnode_msg(file, &msg.u_setn);
  413. else if ((count == OCFS2_CONTROL_MESSAGE_SETVERSION_TOTAL_LEN) &&
  414. !strncmp(msg.tag, OCFS2_CONTROL_MESSAGE_SETVERSION_OP,
  415. OCFS2_CONTROL_MESSAGE_OP_LEN))
  416. ret = ocfs2_control_do_setversion_msg(file, &msg.u_setv);
  417. else if ((count == OCFS2_CONTROL_MESSAGE_DOWN_TOTAL_LEN) &&
  418. !strncmp(msg.tag, OCFS2_CONTROL_MESSAGE_DOWN_OP,
  419. OCFS2_CONTROL_MESSAGE_OP_LEN))
  420. ret = ocfs2_control_do_down_msg(file, &msg.u_down);
  421. else
  422. ret = -EINVAL;
  423. out:
  424. return ret ? ret : count;
  425. }
  426. static ssize_t ocfs2_control_write(struct file *file,
  427. const char __user *buf,
  428. size_t count,
  429. loff_t *ppos)
  430. {
  431. ssize_t ret;
  432. switch (ocfs2_control_get_handshake_state(file)) {
  433. case OCFS2_CONTROL_HANDSHAKE_INVALID:
  434. ret = -EINVAL;
  435. break;
  436. case OCFS2_CONTROL_HANDSHAKE_READ:
  437. ret = ocfs2_control_validate_protocol(file, buf,
  438. count);
  439. break;
  440. case OCFS2_CONTROL_HANDSHAKE_PROTOCOL:
  441. case OCFS2_CONTROL_HANDSHAKE_VALID:
  442. ret = ocfs2_control_message(file, buf, count);
  443. break;
  444. default:
  445. BUG();
  446. ret = -EIO;
  447. break;
  448. }
  449. return ret;
  450. }
  451. /*
  452. * This is a naive version. If we ever have a new protocol, we'll expand
  453. * it. Probably using seq_file.
  454. */
  455. static ssize_t ocfs2_control_read(struct file *file,
  456. char __user *buf,
  457. size_t count,
  458. loff_t *ppos)
  459. {
  460. ssize_t ret;
  461. ret = simple_read_from_buffer(buf, count, ppos,
  462. OCFS2_CONTROL_PROTO, OCFS2_CONTROL_PROTO_LEN);
  463. /* Have we read the whole protocol list? */
  464. if (ret > 0 && *ppos >= OCFS2_CONTROL_PROTO_LEN)
  465. ocfs2_control_set_handshake_state(file,
  466. OCFS2_CONTROL_HANDSHAKE_READ);
  467. return ret;
  468. }
  469. static int ocfs2_control_release(struct inode *inode, struct file *file)
  470. {
  471. struct ocfs2_control_private *p = file->private_data;
  472. mutex_lock(&ocfs2_control_lock);
  473. if (ocfs2_control_get_handshake_state(file) !=
  474. OCFS2_CONTROL_HANDSHAKE_VALID)
  475. goto out;
  476. if (atomic_dec_and_test(&ocfs2_control_opened)) {
  477. if (!list_empty(&ocfs2_live_connection_list)) {
  478. /* XXX: Do bad things! */
  479. printk(KERN_ERR
  480. "ocfs2: Unexpected release of ocfs2_control!\n"
  481. " Loss of cluster connection requires "
  482. "an emergency restart!\n");
  483. emergency_restart();
  484. }
  485. /*
  486. * Last valid close clears the node number and resets
  487. * the locking protocol version
  488. */
  489. ocfs2_control_this_node = -1;
  490. running_proto.pv_major = 0;
  491. running_proto.pv_minor = 0;
  492. }
  493. out:
  494. list_del_init(&p->op_list);
  495. file->private_data = NULL;
  496. mutex_unlock(&ocfs2_control_lock);
  497. kfree(p);
  498. return 0;
  499. }
  500. static int ocfs2_control_open(struct inode *inode, struct file *file)
  501. {
  502. struct ocfs2_control_private *p;
  503. p = kzalloc(sizeof(struct ocfs2_control_private), GFP_KERNEL);
  504. if (!p)
  505. return -ENOMEM;
  506. p->op_this_node = -1;
  507. mutex_lock(&ocfs2_control_lock);
  508. file->private_data = p;
  509. list_add(&p->op_list, &ocfs2_control_private_list);
  510. mutex_unlock(&ocfs2_control_lock);
  511. return 0;
  512. }
  513. static const struct file_operations ocfs2_control_fops = {
  514. .open = ocfs2_control_open,
  515. .release = ocfs2_control_release,
  516. .read = ocfs2_control_read,
  517. .write = ocfs2_control_write,
  518. .owner = THIS_MODULE,
  519. .llseek = default_llseek,
  520. };
  521. static struct miscdevice ocfs2_control_device = {
  522. .minor = MISC_DYNAMIC_MINOR,
  523. .name = "ocfs2_control",
  524. .fops = &ocfs2_control_fops,
  525. };
  526. static int ocfs2_control_init(void)
  527. {
  528. int rc;
  529. atomic_set(&ocfs2_control_opened, 0);
  530. rc = misc_register(&ocfs2_control_device);
  531. if (rc)
  532. printk(KERN_ERR
  533. "ocfs2: Unable to register ocfs2_control device "
  534. "(errno %d)\n",
  535. -rc);
  536. return rc;
  537. }
  538. static void ocfs2_control_exit(void)
  539. {
  540. misc_deregister(&ocfs2_control_device);
  541. }
  542. static void fsdlm_lock_ast_wrapper(void *astarg)
  543. {
  544. struct ocfs2_dlm_lksb *lksb = astarg;
  545. int status = lksb->lksb_fsdlm.sb_status;
  546. /*
  547. * For now we're punting on the issue of other non-standard errors
  548. * where we can't tell if the unlock_ast or lock_ast should be called.
  549. * The main "other error" that's possible is EINVAL which means the
  550. * function was called with invalid args, which shouldn't be possible
  551. * since the caller here is under our control. Other non-standard
  552. * errors probably fall into the same category, or otherwise are fatal
  553. * which means we can't carry on anyway.
  554. */
  555. if (status == -DLM_EUNLOCK || status == -DLM_ECANCEL)
  556. lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, 0);
  557. else
  558. lksb->lksb_conn->cc_proto->lp_lock_ast(lksb);
  559. }
  560. static void fsdlm_blocking_ast_wrapper(void *astarg, int level)
  561. {
  562. struct ocfs2_dlm_lksb *lksb = astarg;
  563. lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level);
  564. }
  565. static int user_dlm_lock(struct ocfs2_cluster_connection *conn,
  566. int mode,
  567. struct ocfs2_dlm_lksb *lksb,
  568. u32 flags,
  569. void *name,
  570. unsigned int namelen)
  571. {
  572. int ret;
  573. if (!lksb->lksb_fsdlm.sb_lvbptr)
  574. lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb +
  575. sizeof(struct dlm_lksb);
  576. ret = dlm_lock(conn->cc_lockspace, mode, &lksb->lksb_fsdlm,
  577. flags|DLM_LKF_NODLCKWT, name, namelen, 0,
  578. fsdlm_lock_ast_wrapper, lksb,
  579. fsdlm_blocking_ast_wrapper);
  580. return ret;
  581. }
  582. static int user_dlm_unlock(struct ocfs2_cluster_connection *conn,
  583. struct ocfs2_dlm_lksb *lksb,
  584. u32 flags)
  585. {
  586. int ret;
  587. ret = dlm_unlock(conn->cc_lockspace, lksb->lksb_fsdlm.sb_lkid,
  588. flags, &lksb->lksb_fsdlm, lksb);
  589. return ret;
  590. }
  591. static int user_dlm_lock_status(struct ocfs2_dlm_lksb *lksb)
  592. {
  593. return lksb->lksb_fsdlm.sb_status;
  594. }
  595. static int user_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb)
  596. {
  597. int invalid = lksb->lksb_fsdlm.sb_flags & DLM_SBF_VALNOTVALID;
  598. return !invalid;
  599. }
  600. static void *user_dlm_lvb(struct ocfs2_dlm_lksb *lksb)
  601. {
  602. if (!lksb->lksb_fsdlm.sb_lvbptr)
  603. lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb +
  604. sizeof(struct dlm_lksb);
  605. return (void *)(lksb->lksb_fsdlm.sb_lvbptr);
  606. }
  607. static void user_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb)
  608. {
  609. }
  610. static int user_plock(struct ocfs2_cluster_connection *conn,
  611. u64 ino,
  612. struct file *file,
  613. int cmd,
  614. struct file_lock *fl)
  615. {
  616. /*
  617. * This more or less just demuxes the plock request into any
  618. * one of three dlm calls.
  619. *
  620. * Internally, fs/dlm will pass these to a misc device, which
  621. * a userspace daemon will read and write to.
  622. *
  623. * For now, cancel requests (which happen internally only),
  624. * are turned into unlocks. Most of this function taken from
  625. * gfs2_lock.
  626. */
  627. if (cmd == F_CANCELLK) {
  628. cmd = F_SETLK;
  629. fl->fl_type = F_UNLCK;
  630. }
  631. if (IS_GETLK(cmd))
  632. return dlm_posix_get(conn->cc_lockspace, ino, file, fl);
  633. else if (fl->fl_type == F_UNLCK)
  634. return dlm_posix_unlock(conn->cc_lockspace, ino, file, fl);
  635. else
  636. return dlm_posix_lock(conn->cc_lockspace, ino, file, cmd, fl);
  637. }
  638. /*
  639. * Compare a requested locking protocol version against the current one.
  640. *
  641. * If the major numbers are different, they are incompatible.
  642. * If the current minor is greater than the request, they are incompatible.
  643. * If the current minor is less than or equal to the request, they are
  644. * compatible, and the requester should run at the current minor version.
  645. */
  646. static int fs_protocol_compare(struct ocfs2_protocol_version *existing,
  647. struct ocfs2_protocol_version *request)
  648. {
  649. if (existing->pv_major != request->pv_major)
  650. return 1;
  651. if (existing->pv_minor > request->pv_minor)
  652. return 1;
  653. if (existing->pv_minor < request->pv_minor)
  654. request->pv_minor = existing->pv_minor;
  655. return 0;
  656. }
  657. static void lvb_to_version(char *lvb, struct ocfs2_protocol_version *ver)
  658. {
  659. struct ocfs2_protocol_version *pv =
  660. (struct ocfs2_protocol_version *)lvb;
  661. /*
  662. * ocfs2_protocol_version has two u8 variables, so we don't
  663. * need any endian conversion.
  664. */
  665. ver->pv_major = pv->pv_major;
  666. ver->pv_minor = pv->pv_minor;
  667. }
  668. static void version_to_lvb(struct ocfs2_protocol_version *ver, char *lvb)
  669. {
  670. struct ocfs2_protocol_version *pv =
  671. (struct ocfs2_protocol_version *)lvb;
  672. /*
  673. * ocfs2_protocol_version has two u8 variables, so we don't
  674. * need any endian conversion.
  675. */
  676. pv->pv_major = ver->pv_major;
  677. pv->pv_minor = ver->pv_minor;
  678. }
  679. static void sync_wait_cb(void *arg)
  680. {
  681. struct ocfs2_cluster_connection *conn = arg;
  682. struct ocfs2_live_connection *lc = conn->cc_private;
  683. complete(&lc->oc_sync_wait);
  684. }
  685. static int sync_unlock(struct ocfs2_cluster_connection *conn,
  686. struct dlm_lksb *lksb, char *name)
  687. {
  688. int error;
  689. struct ocfs2_live_connection *lc = conn->cc_private;
  690. error = dlm_unlock(conn->cc_lockspace, lksb->sb_lkid, 0, lksb, conn);
  691. if (error) {
  692. printk(KERN_ERR "%s lkid %x error %d\n",
  693. name, lksb->sb_lkid, error);
  694. return error;
  695. }
  696. wait_for_completion(&lc->oc_sync_wait);
  697. if (lksb->sb_status != -DLM_EUNLOCK) {
  698. printk(KERN_ERR "%s lkid %x status %d\n",
  699. name, lksb->sb_lkid, lksb->sb_status);
  700. return -1;
  701. }
  702. return 0;
  703. }
  704. static int sync_lock(struct ocfs2_cluster_connection *conn,
  705. int mode, uint32_t flags,
  706. struct dlm_lksb *lksb, char *name)
  707. {
  708. int error, status;
  709. struct ocfs2_live_connection *lc = conn->cc_private;
  710. error = dlm_lock(conn->cc_lockspace, mode, lksb, flags,
  711. name, strlen(name),
  712. 0, sync_wait_cb, conn, NULL);
  713. if (error) {
  714. printk(KERN_ERR "%s lkid %x flags %x mode %d error %d\n",
  715. name, lksb->sb_lkid, flags, mode, error);
  716. return error;
  717. }
  718. wait_for_completion(&lc->oc_sync_wait);
  719. status = lksb->sb_status;
  720. if (status && status != -EAGAIN) {
  721. printk(KERN_ERR "%s lkid %x flags %x mode %d status %d\n",
  722. name, lksb->sb_lkid, flags, mode, status);
  723. }
  724. return status;
  725. }
  726. static int version_lock(struct ocfs2_cluster_connection *conn, int mode,
  727. int flags)
  728. {
  729. struct ocfs2_live_connection *lc = conn->cc_private;
  730. return sync_lock(conn, mode, flags,
  731. &lc->oc_version_lksb, VERSION_LOCK);
  732. }
  733. static int version_unlock(struct ocfs2_cluster_connection *conn)
  734. {
  735. struct ocfs2_live_connection *lc = conn->cc_private;
  736. return sync_unlock(conn, &lc->oc_version_lksb, VERSION_LOCK);
  737. }
  738. /* get_protocol_version()
  739. *
  740. * To exchange ocfs2 versioning, we use the LVB of the version dlm lock.
  741. * The algorithm is:
  742. * 1. Attempt to take the lock in EX mode (non-blocking).
  743. * 2. If successful (which means it is the first mount), write the
  744. * version number and downconvert to PR lock.
  745. * 3. If unsuccessful (returns -EAGAIN), read the version from the LVB after
  746. * taking the PR lock.
  747. */
  748. static int get_protocol_version(struct ocfs2_cluster_connection *conn)
  749. {
  750. int ret;
  751. struct ocfs2_live_connection *lc = conn->cc_private;
  752. struct ocfs2_protocol_version pv;
  753. running_proto.pv_major =
  754. ocfs2_user_plugin.sp_max_proto.pv_major;
  755. running_proto.pv_minor =
  756. ocfs2_user_plugin.sp_max_proto.pv_minor;
  757. lc->oc_version_lksb.sb_lvbptr = lc->oc_lvb;
  758. ret = version_lock(conn, DLM_LOCK_EX,
  759. DLM_LKF_VALBLK|DLM_LKF_NOQUEUE);
  760. if (!ret) {
  761. conn->cc_version.pv_major = running_proto.pv_major;
  762. conn->cc_version.pv_minor = running_proto.pv_minor;
  763. version_to_lvb(&running_proto, lc->oc_lvb);
  764. version_lock(conn, DLM_LOCK_PR, DLM_LKF_CONVERT|DLM_LKF_VALBLK);
  765. } else if (ret == -EAGAIN) {
  766. ret = version_lock(conn, DLM_LOCK_PR, DLM_LKF_VALBLK);
  767. if (ret)
  768. goto out;
  769. lvb_to_version(lc->oc_lvb, &pv);
  770. if ((pv.pv_major != running_proto.pv_major) ||
  771. (pv.pv_minor > running_proto.pv_minor)) {
  772. ret = -EINVAL;
  773. goto out;
  774. }
  775. conn->cc_version.pv_major = pv.pv_major;
  776. conn->cc_version.pv_minor = pv.pv_minor;
  777. }
  778. out:
  779. return ret;
  780. }
  781. static void user_recover_prep(void *arg)
  782. {
  783. }
  784. static void user_recover_slot(void *arg, struct dlm_slot *slot)
  785. {
  786. struct ocfs2_cluster_connection *conn = arg;
  787. printk(KERN_INFO "ocfs2: Node %d/%d down. Initiating recovery.\n",
  788. slot->nodeid, slot->slot);
  789. conn->cc_recovery_handler(slot->nodeid, conn->cc_recovery_data);
  790. }
  791. static void user_recover_done(void *arg, struct dlm_slot *slots,
  792. int num_slots, int our_slot,
  793. uint32_t generation)
  794. {
  795. struct ocfs2_cluster_connection *conn = arg;
  796. struct ocfs2_live_connection *lc = conn->cc_private;
  797. int i;
  798. for (i = 0; i < num_slots; i++)
  799. if (slots[i].slot == our_slot) {
  800. atomic_set(&lc->oc_this_node, slots[i].nodeid);
  801. break;
  802. }
  803. lc->oc_our_slot = our_slot;
  804. wake_up(&lc->oc_wait);
  805. }
  806. static const struct dlm_lockspace_ops ocfs2_ls_ops = {
  807. .recover_prep = user_recover_prep,
  808. .recover_slot = user_recover_slot,
  809. .recover_done = user_recover_done,
  810. };
  811. static int user_cluster_disconnect(struct ocfs2_cluster_connection *conn)
  812. {
  813. version_unlock(conn);
  814. dlm_release_lockspace(conn->cc_lockspace, 2);
  815. conn->cc_lockspace = NULL;
  816. ocfs2_live_connection_drop(conn->cc_private);
  817. conn->cc_private = NULL;
  818. return 0;
  819. }
  820. static int user_cluster_connect(struct ocfs2_cluster_connection *conn)
  821. {
  822. dlm_lockspace_t *fsdlm;
  823. struct ocfs2_live_connection *lc;
  824. int rc, ops_rv;
  825. BUG_ON(conn == NULL);
  826. lc = kzalloc(sizeof(struct ocfs2_live_connection), GFP_KERNEL);
  827. if (!lc)
  828. return -ENOMEM;
  829. init_waitqueue_head(&lc->oc_wait);
  830. init_completion(&lc->oc_sync_wait);
  831. atomic_set(&lc->oc_this_node, 0);
  832. conn->cc_private = lc;
  833. lc->oc_type = NO_CONTROLD;
  834. rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name,
  835. DLM_LSFL_FS | DLM_LSFL_NEWEXCL, DLM_LVB_LEN,
  836. &ocfs2_ls_ops, conn, &ops_rv, &fsdlm);
  837. if (rc) {
  838. if (rc == -EEXIST || rc == -EPROTO)
  839. printk(KERN_ERR "ocfs2: Unable to create the "
  840. "lockspace %s (%d), because a ocfs2-tools "
  841. "program is running on this file system "
  842. "with the same name lockspace\n",
  843. conn->cc_name, rc);
  844. goto out;
  845. }
  846. if (ops_rv == -EOPNOTSUPP) {
  847. lc->oc_type = WITH_CONTROLD;
  848. printk(KERN_NOTICE "ocfs2: You seem to be using an older "
  849. "version of dlm_controld and/or ocfs2-tools."
  850. " Please consider upgrading.\n");
  851. } else if (ops_rv) {
  852. rc = ops_rv;
  853. goto out;
  854. }
  855. conn->cc_lockspace = fsdlm;
  856. rc = ocfs2_live_connection_attach(conn, lc);
  857. if (rc)
  858. goto out;
  859. if (lc->oc_type == NO_CONTROLD) {
  860. rc = get_protocol_version(conn);
  861. if (rc) {
  862. printk(KERN_ERR "ocfs2: Could not determine"
  863. " locking version\n");
  864. user_cluster_disconnect(conn);
  865. goto out;
  866. }
  867. wait_event(lc->oc_wait, (atomic_read(&lc->oc_this_node) > 0));
  868. }
  869. /*
  870. * running_proto must have been set before we allowed any mounts
  871. * to proceed.
  872. */
  873. if (fs_protocol_compare(&running_proto, &conn->cc_version)) {
  874. printk(KERN_ERR
  875. "Unable to mount with fs locking protocol version "
  876. "%u.%u because negotiated protocol is %u.%u\n",
  877. conn->cc_version.pv_major, conn->cc_version.pv_minor,
  878. running_proto.pv_major, running_proto.pv_minor);
  879. rc = -EPROTO;
  880. ocfs2_live_connection_drop(lc);
  881. lc = NULL;
  882. }
  883. out:
  884. if (rc)
  885. kfree(lc);
  886. return rc;
  887. }
  888. static int user_cluster_this_node(struct ocfs2_cluster_connection *conn,
  889. unsigned int *this_node)
  890. {
  891. int rc;
  892. struct ocfs2_live_connection *lc = conn->cc_private;
  893. if (lc->oc_type == WITH_CONTROLD)
  894. rc = ocfs2_control_get_this_node();
  895. else if (lc->oc_type == NO_CONTROLD)
  896. rc = atomic_read(&lc->oc_this_node);
  897. else
  898. rc = -EINVAL;
  899. if (rc < 0)
  900. return rc;
  901. *this_node = rc;
  902. return 0;
  903. }
  904. static struct ocfs2_stack_operations ocfs2_user_plugin_ops = {
  905. .connect = user_cluster_connect,
  906. .disconnect = user_cluster_disconnect,
  907. .this_node = user_cluster_this_node,
  908. .dlm_lock = user_dlm_lock,
  909. .dlm_unlock = user_dlm_unlock,
  910. .lock_status = user_dlm_lock_status,
  911. .lvb_valid = user_dlm_lvb_valid,
  912. .lock_lvb = user_dlm_lvb,
  913. .plock = user_plock,
  914. .dump_lksb = user_dlm_dump_lksb,
  915. };
  916. static struct ocfs2_stack_plugin ocfs2_user_plugin = {
  917. .sp_name = "user",
  918. .sp_ops = &ocfs2_user_plugin_ops,
  919. .sp_owner = THIS_MODULE,
  920. };
  921. static int __init ocfs2_user_plugin_init(void)
  922. {
  923. int rc;
  924. rc = ocfs2_control_init();
  925. if (!rc) {
  926. rc = ocfs2_stack_glue_register(&ocfs2_user_plugin);
  927. if (rc)
  928. ocfs2_control_exit();
  929. }
  930. return rc;
  931. }
  932. static void __exit ocfs2_user_plugin_exit(void)
  933. {
  934. ocfs2_stack_glue_unregister(&ocfs2_user_plugin);
  935. ocfs2_control_exit();
  936. }
  937. MODULE_AUTHOR("Oracle");
  938. MODULE_DESCRIPTION("ocfs2 driver for userspace cluster stacks");
  939. MODULE_LICENSE("GPL");
  940. module_init(ocfs2_user_plugin_init);
  941. module_exit(ocfs2_user_plugin_exit);