libocxl
mmio.c
Go to the documentation of this file.
1 /*
2  * Copyright 2017 International Business Machines
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 // Needed for le32toh() and friends when building against glibc version < 2.20
18 #define _BSD_SOURCE
19 
20 #include "libocxl_internal.h"
21 #include "sys/mman.h"
22 #include "errno.h"
23 #include <sys/types.h>
24 #include <sys/stat.h>
25 #include <fcntl.h>
26 #include <unistd.h>
27 #include <string.h>
28 #include <stdlib.h>
29 
55 static ocxl_err register_mmio(ocxl_afu *afu, void *addr, size_t size, ocxl_mmio_type type, ocxl_mmio_h *handle)
56 {
57  int available_mmio = -1;
58 
59  // Look for an available MMIO region that has been unmapped
60  for (uint16_t mmio = 0; mmio < afu->mmio_count; mmio++) {
61  if (!afu->mmios[mmio].start) {
62  available_mmio = mmio;
63  break;
64  }
65  }
66 
67  if (available_mmio == -1) {
68  if (afu->mmio_count == afu->mmio_max_count) {
69  ocxl_err rc = grow_buffer(afu, (void **)&afu->mmios, &afu->mmio_max_count, sizeof(ocxl_mmio_area), INITIAL_MMIO_COUNT);
70  if (rc != OCXL_OK) {
71  errmsg(afu, rc, "Could not grow MMIO buffer");
72  return rc;
73  }
74  }
75 
76  available_mmio = afu->mmio_count++;
77  }
78 
79  afu->mmios[available_mmio].start = addr;
80  afu->mmios[available_mmio].length = size;
81  afu->mmios[available_mmio].type = type;
82  afu->mmios[available_mmio].afu = afu;
83 
84  *handle = &afu->mmios[available_mmio];
85 
86  TRACE(afu, "Mapped %ld bytes of %s MMIO at %p",
87  size, type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID", addr);
88 
89  return OCXL_OK;
90 }
91 
99 ocxl_err global_mmio_open(ocxl_afu *afu)
100 {
101  char path[PATH_MAX + 1];
102  int length = snprintf(path, sizeof(path), "%s/global_mmio_area", afu->sysfs_path);
103  if (length >= (int)sizeof(path)) {
104  ocxl_err rc = OCXL_NO_DEV;
105  errmsg(afu, rc, "global MMIO path truncated");
106  return rc;
107  }
108 
109  int fd = open(path, O_RDWR | O_CLOEXEC);
110  if (fd < 0) {
111  ocxl_err rc = OCXL_NO_DEV;
112  errmsg(afu, rc, "Could not open global MMIO '%s': Error %d: %s", path, errno, strerror(errno));
113  return rc;
114  }
115 
116  afu->global_mmio_fd = fd;
117 
118  return OCXL_OK;
119 }
120 
144 static ocxl_err global_mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset,
145  ocxl_mmio_h *region) // static function extraction hack
146 {
147  if (afu->global_mmio.length == 0) {
148  ocxl_err rc = OCXL_NO_MEM;
149  errmsg(afu, rc, "Cannot map Global MMIO as there is 0 bytes allocated by the AFU");
150  return rc;
151  }
152 
153  if (flags) {
155  errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
156  return rc;
157  }
158 
159  void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->global_mmio_fd, offset);
160  if (addr == MAP_FAILED) {
161  ocxl_err rc = OCXL_NO_MEM;
162  errmsg(afu, rc, "Could not map global MMIO, %d: %s", errno, strerror(errno));
163  return rc;
164  }
165 
166  ocxl_mmio_h mmio_region;
167  ocxl_err rc = register_mmio(afu, addr, size, OCXL_GLOBAL_MMIO, &mmio_region);
168  if (rc != OCXL_OK) {
169  errmsg(afu, rc, "Could not register global MMIO region");
170  munmap(addr, size);
171  return rc;
172  }
173 
174  *region = mmio_region;
175 
176  return OCXL_OK;
177 }
178 
202 static ocxl_err mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
203 {
204  if (flags) {
206  errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
207  return rc;
208  }
209 
210  if (afu->fd < 0) {
212  errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been opened");
213  return rc;
214  }
215 
216  if (!afu->attached) {
218  errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been attached");
219  return rc;
220  }
221 
222  void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->fd, offset);
223  if (addr == MAP_FAILED) {
224  ocxl_err rc = OCXL_NO_MEM;
225  errmsg(afu, rc, "Could not map per-PASID MMIO: %d: %s", errno, strerror(errno));
226  return rc;
227  }
228 
229  ocxl_mmio_h mmio_region;
230  ocxl_err rc = register_mmio(afu, addr, size, OCXL_PER_PASID_MMIO, &mmio_region);
231  if (rc != OCXL_OK) {
232  errmsg(afu, rc, "Could not register global MMIO region", afu->identifier.afu_name);
233  munmap(addr, size);
234  return rc;
235  }
236 
237  *region = mmio_region;
238 
239  return OCXL_OK;
240 }
241 
263 ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags,
264  off_t offset, ocxl_mmio_h *region)
265 {
267 
268  if (size == 0) {
269  switch (type) {
270  case OCXL_PER_PASID_MMIO:
271  size = afu->per_pasid_mmio.length;
272  break;
273  case OCXL_GLOBAL_MMIO:
274  size = afu->global_mmio.length;
275  break;
276  }
277 
278  size -= offset;
279  }
280 
281  switch (type) {
282  case OCXL_GLOBAL_MMIO:
283  if (offset + size > afu->global_mmio.length) {
284  rc = OCXL_NO_MEM;
285  errmsg(afu, rc, "Offset(%#x) + size(%#x) of global MMIO map request exceeds available size of %#x",
286  offset, size, afu->global_mmio.length);
287  return rc;
288  }
289  return global_mmio_map(afu, size, prot, flags, offset, region);
290 
291  case OCXL_PER_PASID_MMIO:
292  if (offset + size > afu->per_pasid_mmio.length) {
293  rc = OCXL_NO_MEM;
294  errmsg(afu, rc, "Offset(%#x) + size(%#x) of per-pasid MMIO map request exceeds available size of %#x",
295  offset, size, afu->global_mmio.length);
296  return rc;
297  }
298  return mmio_map(afu, size, prot, flags, offset, region);
299 
300  default:
301  errmsg(afu, rc, "Unknown MMIO type %d", type);
302  return rc;
303  }
304 }
305 
325 {
326  return ocxl_mmio_map_advanced(afu, type, 0, PROT_READ | PROT_WRITE, 0, 0, region);
327 }
328 
337 {
338  if (!region->start) {
339  return;
340  }
341 
342  munmap(region->start, region->length);
343  region->start = NULL;
344 }
345 
361 {
362  switch (type) {
363  case OCXL_GLOBAL_MMIO:
364  return afu->global_mmio_fd;
365 
366  case OCXL_PER_PASID_MMIO:
367  return afu->fd;
368 
369  default:
370  errmsg(afu, OCXL_INVALID_ARGS, "Unknown MMIO type %d", type);
371  return -1;
372  }
373 }
374 
384 {
385  switch(type) {
386  case OCXL_GLOBAL_MMIO:
387  return afu->global_mmio.length;
388 
389  case OCXL_PER_PASID_MMIO:
390  return afu->per_pasid_mmio.length;
391 
392  default:
393  errmsg(afu, OCXL_INVALID_ARGS, "Invalid MMIO area requested '%d'", type);
394  return 0;
395  }
396 }
397 
398 
411 ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
412 {
413  if (!region->start) {
415  errmsg(region->afu, rc, "MMIO region has already been unmapped");
416  return rc;
417  }
418 
419  *address = region->start;
420  *size = region->length;
421 
422  return OCXL_OK;
423 }
424 
425 
437 inline static ocxl_err mmio_check(ocxl_mmio_h region, off_t offset, size_t size)
438 {
439  if (!region) {
441  errmsg(NULL, rc, "MMIO region is invalid");
442  return rc;
443  }
444 
445  if (!region->start) {
447  errmsg(region->afu, rc, "MMIO region has already been unmapped");
448  return rc;
449  }
450 
451  if (offset >= (off_t)(region->length - (size - 1))) {
453  errmsg(region->afu, rc, "%s MMIO access of 0x%016lx exceeds limit of 0x%016lx",
454  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
455  offset, region->length);
456  return rc;
457  }
458 
459  return OCXL_OK;
460 }
461 
480 inline static ocxl_err mmio_read32_native(ocxl_mmio_h region, off_t offset, uint32_t *out)
481 {
482  ocxl_err ret = mmio_check(region, offset, 4);
483  if (ret != OCXL_OK) {
484  return ret;
485  }
486 
487  __sync_synchronize();
488  *out = *(volatile uint32_t *)(region->start + offset);
489  __sync_synchronize();
490 
491  TRACE(region->afu, "%s MMIO Read32@0x%04lx=0x%08x",
492  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
493  offset, *out);
494 
495  return OCXL_OK;
496 }
497 
516 inline static ocxl_err mmio_read64_native(ocxl_mmio_h region, off_t offset, uint64_t *out)
517 {
518  ocxl_err ret = mmio_check(region, offset, 8);
519  if (ret != OCXL_OK) {
520  return ret;
521  }
522 
523  __sync_synchronize();
524  *out = *(volatile uint64_t *)(region->start + offset);
525  __sync_synchronize();
526 
527  TRACE(region->afu, "%s MMIO Read64@0x%04lx=0x%016lx",
528  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
529  offset, *out);
530 
531  return OCXL_OK;
532 }
533 
554 inline static ocxl_err mmio_write32_native(ocxl_mmio_h region, off_t offset, uint32_t value)
555 {
556  ocxl_err ret = mmio_check(region, offset, 4);
557  if (ret != OCXL_OK) {
558  return ret;
559  }
560 
561  TRACE(region->afu, "%s MMIO Write32@0x%04lx=0x%08x",
562  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
563  offset, value);
564 
565  volatile uint32_t *addr = (uint32_t *)(region->start + offset);
566 
567  __sync_synchronize();
568  *addr = value;
569  __sync_synchronize();
570 
571  return OCXL_OK;
572 }
573 
592 inline static ocxl_err mmio_write64_native(ocxl_mmio_h region, off_t offset, uint64_t value)
593 {
594  ocxl_err ret = mmio_check(region, offset, 8);
595  if (ret != OCXL_OK) {
596  return ret;
597  }
598 
599  TRACE(region->afu, "%s MMIO Write64@0x%04lx=0x%016lx",
600  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
601  offset, value);
602 
603  volatile uint64_t *addr = (uint64_t *)(region->start + offset);
604 
605  __sync_synchronize();
606  *addr = value;
607  __sync_synchronize();
608 
609  return OCXL_OK;
610 }
611 
630 ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
631 {
632  uint32_t val;
633  ocxl_err ret = mmio_read32_native(mmio, offset, &val);
634 
635  if (UNLIKELY(ret != OCXL_OK)) {
636  return ret;
637  }
638 
639  switch (endian) {
641  *out = be32toh(val);
642  break;
643 
645  *out = le32toh(val);
646  break;
647  default:
648  *out = val;
649  break;
650  }
651 
652  return OCXL_OK;
653 }
654 
674 ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
675 {
676  uint64_t val;
677  ocxl_err ret = mmio_read64_native(mmio, offset, &val);
678 
679  if (UNLIKELY(ret != OCXL_OK)) {
680  return ret;
681  }
682 
683  switch (endian) {
685  *out = be64toh(val);
686  break;
687 
689  *out = le64toh(val);
690  break;
691  default:
692  *out = val;
693  break;
694  }
695 
696  return OCXL_OK;
697 }
698 
718 ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
719 {
720  switch (endian) {
722  value = htobe32(value);
723  break;
724 
726  value = htole32(value);
727  break;
728  default:
729  break;
730  }
731 
732  return mmio_write32_native(mmio, offset, value);
733 }
734 
753 ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
754 {
755  switch (endian) {
757  value = htobe64(value);
758  break;
759 
761  value = htole64(value);
762  break;
763  default:
764  break;
765  }
766 
767  return mmio_write64_native(mmio, offset, value);
768 }
769 
770 
size_t ocxl_mmio_size(ocxl_afu_h afu, ocxl_mmio_type type)
Get the size of an MMIO region for an AFU.
Definition: mmio.c:383
ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
Convert endianness and write a 64-bit value to an AFU's MMIO region.
Definition: mmio.c:753
int ocxl_mmio_get_fd(ocxl_afu_h afu, ocxl_mmio_type type)
Get a file descriptor for an MMIO area of an AFU.
Definition: mmio.c:360
ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition: mmio.c:263
ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
Read a 64-bit value from an AFU's MMIO region & convert endianness.
Definition: mmio.c:674
void ocxl_mmio_unmap(ocxl_mmio_h region)
Unmap an MMIO region from an AFU.
Definition: mmio.c:336
ocxl_err ocxl_mmio_map(ocxl_afu_h afu, ocxl_mmio_type type, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition: mmio.c:324
ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
Convert endianness and write a 32-bit value to an AFU's MMIO region.
Definition: mmio.c:718
ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
Read a 32-bit value from an AFU's MMIO region & convert endianness.
Definition: mmio.c:630
ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
Get the address & size of a mapped MMIO region.
Definition: mmio.c:411
ocxl_err global_mmio_open(ocxl_afu *afu)
Open the global MMIO descriptor on an AFU.
Definition: mmio.c:99
ocxl_err
Potential return values from ocxl_* functions.
Definition: libocxl.h:92
@ OCXL_NO_DEV
The OpenCAPI device is not available.
Definition: libocxl.h:95
@ OCXL_INVALID_ARGS
One or more arguments are invalid.
Definition: libocxl.h:102
@ OCXL_OUT_OF_BOUNDS
The action requested falls outside the permitted area.
Definition: libocxl.h:100
@ OCXL_OK
The call succeeded.
Definition: libocxl.h:93
@ OCXL_NO_MEM
An out of memory error occurred.
Definition: libocxl.h:94
@ OCXL_NO_CONTEXT
The call requires an open context on the AFU.
Definition: libocxl.h:96
ocxl_mmio_type
Defines the type of an MMIO area.
Definition: libocxl.h:56
@ OCXL_GLOBAL_MMIO
Definition: libocxl.h:57
@ OCXL_PER_PASID_MMIO
Definition: libocxl.h:58
struct ocxl_afu * ocxl_afu_h
A handle for an AFU.
Definition: libocxl.h:74
struct ocxl_mmio_area * ocxl_mmio_h
A handle for an MMIO region on an AFU.
Definition: libocxl.h:86
ocxl_endian
Defines the endianness of an AFU MMIO area.
Definition: libocxl.h:47
@ OCXL_MMIO_LITTLE_ENDIAN
AFU data is little-endian.
Definition: libocxl.h:49
@ OCXL_MMIO_BIG_ENDIAN
AFU data is big-endian.
Definition: libocxl.h:48